2024-11-28 01:40:43,850 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 01:40:43,872 main DEBUG Took 0.018867 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 01:40:43,872 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 01:40:43,873 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 01:40:43,874 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 01:40:43,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,890 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 01:40:43,918 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,920 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,922 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,923 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,926 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,926 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,927 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,928 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,928 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,930 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,930 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,931 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,931 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,933 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,933 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,934 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,935 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,935 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,936 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:40:43,937 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,938 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 01:40:43,940 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:40:43,942 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 01:40:43,945 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 01:40:43,945 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 01:40:43,947 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 01:40:43,947 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 01:40:43,960 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 01:40:43,963 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 01:40:43,966 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 01:40:43,966 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 01:40:43,967 main DEBUG createAppenders(={Console}) 2024-11-28 01:40:43,967 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-28 01:40:43,968 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 01:40:43,968 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-28 01:40:43,969 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 01:40:43,969 main DEBUG OutputStream closed 2024-11-28 01:40:43,970 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 01:40:43,970 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 01:40:43,971 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-28 01:40:44,082 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 01:40:44,085 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 01:40:44,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 01:40:44,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 01:40:44,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 01:40:44,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 01:40:44,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 01:40:44,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 01:40:44,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 01:40:44,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 01:40:44,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 01:40:44,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 01:40:44,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 01:40:44,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 01:40:44,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 01:40:44,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 01:40:44,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 01:40:44,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 01:40:44,095 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 01:40:44,095 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-28 01:40:44,096 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 01:40:44,097 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-28T01:40:44,522 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4 2024-11-28 01:40:44,526 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 01:40:44,526 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T01:40:44,539 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-28T01:40:44,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T01:40:44,621 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d, deleteOnExit=true 2024-11-28T01:40:44,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T01:40:44,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/test.cache.data in system properties and HBase conf 2024-11-28T01:40:44,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T01:40:44,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/hadoop.log.dir in system properties and HBase conf 2024-11-28T01:40:44,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T01:40:44,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T01:40:44,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T01:40:44,755 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T01:40:44,928 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:40:44,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T01:40:44,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T01:40:44,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T01:40:44,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T01:40:44,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T01:40:44,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T01:40:44,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T01:40:44,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T01:40:44,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T01:40:44,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/nfs.dump.dir in system properties and HBase conf 2024-11-28T01:40:44,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/java.io.tmpdir in system properties and HBase conf 2024-11-28T01:40:44,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T01:40:44,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T01:40:44,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T01:40:45,967 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T01:40:46,067 INFO [Time-limited test {}] log.Log(170): Logging initialized @3274ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T01:40:46,172 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T01:40:46,253 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T01:40:46,290 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T01:40:46,291 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T01:40:46,292 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T01:40:46,308 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T01:40:46,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/hadoop.log.dir/,AVAILABLE} 2024-11-28T01:40:46,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T01:40:46,568 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/java.io.tmpdir/jetty-localhost-39673-hadoop-hdfs-3_4_1-tests_jar-_-any-1676866689643731736/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T01:40:46,577 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39673} 2024-11-28T01:40:46,578 INFO [Time-limited test {}] server.Server(415): Started @3786ms 2024-11-28T01:40:47,029 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T01:40:47,037 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T01:40:47,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T01:40:47,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T01:40:47,038 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T01:40:47,039 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/hadoop.log.dir/,AVAILABLE} 2024-11-28T01:40:47,040 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T01:40:47,167 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/java.io.tmpdir/jetty-localhost-46143-hadoop-hdfs-3_4_1-tests_jar-_-any-15137851035796258200/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T01:40:47,168 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:46143} 2024-11-28T01:40:47,168 INFO [Time-limited test {}] server.Server(415): Started @4377ms 2024-11-28T01:40:47,229 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T01:40:47,788 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/data/data1/current/BP-129411190-172.17.0.2-1732758045693/current, will proceed with Du for space computation calculation, 2024-11-28T01:40:47,790 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/data/data2/current/BP-129411190-172.17.0.2-1732758045693/current, will proceed with Du for space computation calculation, 2024-11-28T01:40:47,848 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T01:40:47,927 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1177790b5584a60d with lease ID 0xdd79a4769240ed6a: Processing first storage report for DS-be377e90-dff8-4065-ac90-9e54563e4d4a from datanode DatanodeRegistration(127.0.0.1:35065, datanodeUuid=5222bf73-d4c7-4850-a6fb-744c437557f8, infoPort=44411, infoSecurePort=0, ipcPort=34745, storageInfo=lv=-57;cid=testClusterID;nsid=641720931;c=1732758045693) 2024-11-28T01:40:47,928 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1177790b5584a60d with lease ID 0xdd79a4769240ed6a: from storage DS-be377e90-dff8-4065-ac90-9e54563e4d4a node DatanodeRegistration(127.0.0.1:35065, datanodeUuid=5222bf73-d4c7-4850-a6fb-744c437557f8, infoPort=44411, infoSecurePort=0, ipcPort=34745, storageInfo=lv=-57;cid=testClusterID;nsid=641720931;c=1732758045693), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-28T01:40:47,929 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1177790b5584a60d with lease ID 0xdd79a4769240ed6a: Processing first storage report for DS-2f2bddf0-ff8b-42e4-b55a-98a231e83d29 from datanode DatanodeRegistration(127.0.0.1:35065, datanodeUuid=5222bf73-d4c7-4850-a6fb-744c437557f8, infoPort=44411, infoSecurePort=0, ipcPort=34745, storageInfo=lv=-57;cid=testClusterID;nsid=641720931;c=1732758045693) 2024-11-28T01:40:47,929 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1177790b5584a60d with lease ID 0xdd79a4769240ed6a: from storage DS-2f2bddf0-ff8b-42e4-b55a-98a231e83d29 node DatanodeRegistration(127.0.0.1:35065, datanodeUuid=5222bf73-d4c7-4850-a6fb-744c437557f8, infoPort=44411, infoSecurePort=0, ipcPort=34745, storageInfo=lv=-57;cid=testClusterID;nsid=641720931;c=1732758045693), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T01:40:48,017 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4 2024-11-28T01:40:48,150 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/zookeeper_0, clientPort=63369, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T01:40:48,168 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63369 2024-11-28T01:40:48,185 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T01:40:48,189 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T01:40:48,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741825_1001 (size=7) 2024-11-28T01:40:48,926 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec with version=8 2024-11-28T01:40:48,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/hbase-staging 2024-11-28T01:40:49,028 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T01:40:49,353 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:40:49,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:40:49,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:40:49,378 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:40:49,378 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:40:49,379 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:40:49,565 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T01:40:49,634 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T01:40:49,644 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T01:40:49,648 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:40:49,681 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1552 (auto-detected) 2024-11-28T01:40:49,683 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T01:40:49,707 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45903 2024-11-28T01:40:49,739 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45903 connecting to ZooKeeper ensemble=127.0.0.1:63369 2024-11-28T01:40:49,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459030x0, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:40:49,793 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45903-0x1003bc946520000 connected 2024-11-28T01:40:49,849 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T01:40:49,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T01:40:49,865 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:40:49,871 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec, hbase.cluster.distributed=false 2024-11-28T01:40:49,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:40:49,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45903 2024-11-28T01:40:49,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45903 2024-11-28T01:40:49,921 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45903 2024-11-28T01:40:49,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45903 2024-11-28T01:40:49,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45903 2024-11-28T01:40:50,075 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:40:50,077 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:40:50,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:40:50,078 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:40:50,079 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:40:50,079 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:40:50,083 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T01:40:50,086 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:40:50,087 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42371 2024-11-28T01:40:50,090 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42371 connecting to ZooKeeper ensemble=127.0.0.1:63369 2024-11-28T01:40:50,092 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T01:40:50,096 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T01:40:50,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:423710x0, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:40:50,111 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:423710x0, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:40:50,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42371-0x1003bc946520001 connected 2024-11-28T01:40:50,116 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T01:40:50,125 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T01:40:50,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T01:40:50,137 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:40:50,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42371 2024-11-28T01:40:50,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42371 2024-11-28T01:40:50,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42371 2024-11-28T01:40:50,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42371 2024-11-28T01:40:50,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42371 2024-11-28T01:40:50,162 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9df3f25f5867:45903 2024-11-28T01:40:50,163 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9df3f25f5867,45903,1732758049085 2024-11-28T01:40:50,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:40:50,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:40:50,181 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9df3f25f5867,45903,1732758049085 2024-11-28T01:40:50,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T01:40:50,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:50,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:50,211 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T01:40:50,212 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9df3f25f5867,45903,1732758049085 from backup master directory 2024-11-28T01:40:50,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:40:50,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9df3f25f5867,45903,1732758049085 2024-11-28T01:40:50,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:40:50,217 WARN [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:40:50,217 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9df3f25f5867,45903,1732758049085 2024-11-28T01:40:50,220 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T01:40:50,222 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T01:40:50,293 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/hbase.id] with ID: fb45315c-53b7-46a3-81e7-db91cf2c055a 2024-11-28T01:40:50,294 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/.tmp/hbase.id 2024-11-28T01:40:50,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741826_1002 (size=42) 2024-11-28T01:40:50,310 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/.tmp/hbase.id]:[hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/hbase.id] 2024-11-28T01:40:50,362 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T01:40:50,369 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T01:40:50,390 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-28T01:40:50,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:50,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:50,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741827_1003 (size=196) 2024-11-28T01:40:50,433 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:40:50,437 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T01:40:50,456 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:50,461 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:40:50,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741828_1004 (size=1189) 2024-11-28T01:40:50,539 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store 2024-11-28T01:40:50,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741829_1005 (size=34) 2024-11-28T01:40:50,586 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T01:40:50,590 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:40:50,592 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T01:40:50,592 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:40:50,593 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:40:50,594 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T01:40:50,595 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:40:50,595 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:40:50,597 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732758050592Disabling compacts and flushes for region at 1732758050592Disabling writes for close at 1732758050594 (+2 ms)Writing region close event to WAL at 1732758050595 (+1 ms)Closed at 1732758050595 2024-11-28T01:40:50,600 WARN [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/.initializing 2024-11-28T01:40:50,600 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/WALs/9df3f25f5867,45903,1732758049085 2024-11-28T01:40:50,612 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:40:50,632 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9df3f25f5867%2C45903%2C1732758049085, suffix=, logDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/WALs/9df3f25f5867,45903,1732758049085, archiveDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/oldWALs, maxLogs=10 2024-11-28T01:40:50,662 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/WALs/9df3f25f5867,45903,1732758049085/9df3f25f5867%2C45903%2C1732758049085.1732758050639, exclude list is [], retry=0 2024-11-28T01:40:50,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35065,DS-be377e90-dff8-4065-ac90-9e54563e4d4a,DISK] 2024-11-28T01:40:50,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T01:40:50,736 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/WALs/9df3f25f5867,45903,1732758049085/9df3f25f5867%2C45903%2C1732758049085.1732758050639 2024-11-28T01:40:50,738 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44411:44411)] 2024-11-28T01:40:50,738 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:40:50,739 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:40:50,742 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,743 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T01:40:50,816 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:50,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:50,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T01:40:50,824 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:50,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:40:50,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T01:40:50,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:50,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:40:50,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T01:40:50,832 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:50,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:40:50,834 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,837 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,839 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,843 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,844 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,847 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:40:50,851 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:40:50,855 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:40:50,856 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73846262, jitterRate=0.10039505362510681}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:40:50,863 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732758050756Initializing all the Stores at 1732758050758 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758050759 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758050759Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758050760 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758050760Cleaning up temporary data from old regions at 1732758050844 (+84 ms)Region opened successfully at 1732758050863 (+19 ms) 2024-11-28T01:40:50,864 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T01:40:50,898 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77884a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:40:50,930 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T01:40:50,942 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T01:40:50,942 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T01:40:50,945 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T01:40:50,946 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T01:40:50,951 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-28T01:40:50,952 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T01:40:50,977 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T01:40:50,987 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T01:40:50,989 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T01:40:50,991 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T01:40:50,993 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T01:40:50,994 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T01:40:50,997 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T01:40:51,000 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T01:40:51,002 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T01:40:51,003 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T01:40:51,005 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T01:40:51,023 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T01:40:51,025 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T01:40:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:40:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:40:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,033 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9df3f25f5867,45903,1732758049085, sessionid=0x1003bc946520000, setting cluster-up flag (Was=false) 2024-11-28T01:40:51,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,054 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T01:40:51,056 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9df3f25f5867,45903,1732758049085 2024-11-28T01:40:51,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,067 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T01:40:51,068 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9df3f25f5867,45903,1732758049085 2024-11-28T01:40:51,075 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T01:40:51,148 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(746): ClusterId : fb45315c-53b7-46a3-81e7-db91cf2c055a 2024-11-28T01:40:51,151 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T01:40:51,155 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T01:40:51,161 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T01:40:51,161 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T01:40:51,166 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T01:40:51,166 DEBUG [RS:0;9df3f25f5867:42371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c6adb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:40:51,168 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T01:40:51,178 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T01:40:51,186 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9df3f25f5867:42371 2024-11-28T01:40:51,190 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T01:40:51,190 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T01:40:51,190 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T01:40:51,190 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9df3f25f5867,45903,1732758049085 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T01:40:51,193 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,45903,1732758049085 with port=42371, startcode=1732758050017 2024-11-28T01:40:51,199 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:40:51,199 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:40:51,200 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:40:51,200 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:40:51,200 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9df3f25f5867:0, corePoolSize=10, maxPoolSize=10 2024-11-28T01:40:51,201 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,201 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:40:51,201 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,204 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732758081204 2024-11-28T01:40:51,206 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T01:40:51,207 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T01:40:51,208 DEBUG [RS:0;9df3f25f5867:42371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T01:40:51,212 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T01:40:51,212 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T01:40:51,213 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:40:51,213 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T01:40:51,213 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T01:40:51,213 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T01:40:51,214 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,220 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T01:40:51,222 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T01:40:51,223 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T01:40:51,225 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:51,225 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T01:40:51,232 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T01:40:51,232 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T01:40:51,249 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758051234,5,FailOnTimeoutGroup] 2024-11-28T01:40:51,252 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758051249,5,FailOnTimeoutGroup] 2024-11-28T01:40:51,252 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,253 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T01:40:51,254 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,255 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741831_1007 (size=1321) 2024-11-28T01:40:51,274 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T01:40:51,275 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:40:51,277 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43405, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T01:40:51,287 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45903 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9df3f25f5867,42371,1732758050017 2024-11-28T01:40:51,290 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45903 {}] master.ServerManager(517): Registering regionserver=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:51,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741832_1008 (size=32) 2024-11-28T01:40:51,311 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:40:51,312 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35107 2024-11-28T01:40:51,312 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T01:40:51,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:40:51,319 DEBUG [RS:0;9df3f25f5867:42371 {}] zookeeper.ZKUtil(111): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9df3f25f5867,42371,1732758050017 2024-11-28T01:40:51,320 WARN [RS:0;9df3f25f5867:42371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:40:51,320 INFO [RS:0;9df3f25f5867:42371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:40:51,320 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/WALs/9df3f25f5867,42371,1732758050017 2024-11-28T01:40:51,322 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9df3f25f5867,42371,1732758050017] 2024-11-28T01:40:51,356 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T01:40:51,375 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T01:40:51,380 INFO [RS:0;9df3f25f5867:42371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T01:40:51,380 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,381 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T01:40:51,388 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T01:40:51,389 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,390 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,390 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,390 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,390 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,391 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,391 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:40:51,391 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,391 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,392 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,392 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,392 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,392 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:40:51,392 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:40:51,393 DEBUG [RS:0;9df3f25f5867:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:40:51,394 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,394 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,394 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,395 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,395 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,395 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,42371,1732758050017-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:40:51,416 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T01:40:51,418 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,42371,1732758050017-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,419 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,419 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.Replication(171): 9df3f25f5867,42371,1732758050017 started 2024-11-28T01:40:51,440 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:51,440 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1482): Serving as 9df3f25f5867,42371,1732758050017, RpcServer on 9df3f25f5867/172.17.0.2:42371, sessionid=0x1003bc946520001 2024-11-28T01:40:51,441 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T01:40:51,441 DEBUG [RS:0;9df3f25f5867:42371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9df3f25f5867,42371,1732758050017 2024-11-28T01:40:51,441 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,42371,1732758050017' 2024-11-28T01:40:51,441 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T01:40:51,443 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T01:40:51,444 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T01:40:51,444 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T01:40:51,444 DEBUG [RS:0;9df3f25f5867:42371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9df3f25f5867,42371,1732758050017 2024-11-28T01:40:51,444 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,42371,1732758050017' 2024-11-28T01:40:51,444 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T01:40:51,445 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T01:40:51,446 DEBUG [RS:0;9df3f25f5867:42371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T01:40:51,446 INFO [RS:0;9df3f25f5867:42371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T01:40:51,446 INFO [RS:0;9df3f25f5867:42371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T01:40:51,553 INFO [RS:0;9df3f25f5867:42371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:40:51,557 INFO [RS:0;9df3f25f5867:42371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9df3f25f5867%2C42371%2C1732758050017, suffix=, logDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/WALs/9df3f25f5867,42371,1732758050017, archiveDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/oldWALs, maxLogs=32 2024-11-28T01:40:51,574 DEBUG [RS:0;9df3f25f5867:42371 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/WALs/9df3f25f5867,42371,1732758050017/9df3f25f5867%2C42371%2C1732758050017.1732758051559, exclude list is [], retry=0 2024-11-28T01:40:51,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35065,DS-be377e90-dff8-4065-ac90-9e54563e4d4a,DISK] 2024-11-28T01:40:51,582 INFO [RS:0;9df3f25f5867:42371 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/WALs/9df3f25f5867,42371,1732758050017/9df3f25f5867%2C42371%2C1732758050017.1732758051559 2024-11-28T01:40:51,583 DEBUG [RS:0;9df3f25f5867:42371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44411:44411)] 2024-11-28T01:40:51,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:40:51,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T01:40:51,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T01:40:51,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:51,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:51,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T01:40:51,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T01:40:51,713 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:51,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:51,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T01:40:51,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T01:40:51,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:51,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:51,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T01:40:51,721 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T01:40:51,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:51,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:51,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T01:40:51,724 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740 2024-11-28T01:40:51,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740 2024-11-28T01:40:51,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T01:40:51,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T01:40:51,730 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:40:51,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T01:40:51,737 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:40:51,738 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65034026, jitterRate=-0.030917495489120483}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:40:51,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732758051704Initializing all the Stores at 1732758051706 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758051706Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758051706Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758051706Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758051706Cleaning up temporary data from old regions at 1732758051728 (+22 ms)Region opened successfully at 1732758051741 (+13 ms) 2024-11-28T01:40:51,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T01:40:51,741 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T01:40:51,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T01:40:51,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T01:40:51,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T01:40:51,744 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T01:40:51,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732758051741Disabling compacts and flushes for region at 1732758051741Disabling writes for close at 1732758051742 (+1 ms)Writing region close event to WAL at 1732758051743 (+1 ms)Closed at 1732758051744 (+1 ms) 2024-11-28T01:40:51,749 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:40:51,749 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T01:40:51,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T01:40:51,768 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:40:51,771 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T01:40:51,926 DEBUG [9df3f25f5867:45903 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T01:40:51,951 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:51,964 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,42371,1732758050017, state=OPENING 2024-11-28T01:40:51,971 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T01:40:51,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:40:51,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:40:51,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:40:51,980 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:40:51,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:40:52,167 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T01:40:52,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T01:40:52,184 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T01:40:52,184 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:40:52,185 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T01:40:52,189 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9df3f25f5867%2C42371%2C1732758050017.meta, suffix=.meta, logDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/WALs/9df3f25f5867,42371,1732758050017, archiveDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/oldWALs, maxLogs=32 2024-11-28T01:40:52,206 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/WALs/9df3f25f5867,42371,1732758050017/9df3f25f5867%2C42371%2C1732758050017.meta.1732758052190.meta, exclude list is [], retry=0 2024-11-28T01:40:52,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35065,DS-be377e90-dff8-4065-ac90-9e54563e4d4a,DISK] 2024-11-28T01:40:52,214 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/WALs/9df3f25f5867,42371,1732758050017/9df3f25f5867%2C42371%2C1732758050017.meta.1732758052190.meta 2024-11-28T01:40:52,215 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44411:44411)] 2024-11-28T01:40:52,215 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:40:52,217 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T01:40:52,221 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T01:40:52,228 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T01:40:52,234 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T01:40:52,235 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:40:52,235 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T01:40:52,236 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T01:40:52,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T01:40:52,242 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T01:40:52,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:52,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:52,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T01:40:52,245 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T01:40:52,245 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:52,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:52,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T01:40:52,248 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T01:40:52,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:52,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:52,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T01:40:52,250 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T01:40:52,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:52,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:40:52,252 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T01:40:52,253 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740 2024-11-28T01:40:52,255 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740 2024-11-28T01:40:52,258 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T01:40:52,258 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T01:40:52,259 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:40:52,261 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T01:40:52,263 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66145259, jitterRate=-0.01435883343219757}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:40:52,264 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T01:40:52,265 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732758052236Writing region info on filesystem at 1732758052237 (+1 ms)Initializing all the Stores at 1732758052239 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758052239Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758052240 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758052240Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758052240Cleaning up temporary data from old regions at 1732758052258 (+18 ms)Running coprocessor post-open hooks at 1732758052264 (+6 ms)Region opened successfully at 1732758052265 (+1 ms) 2024-11-28T01:40:52,275 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732758052155 2024-11-28T01:40:52,292 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T01:40:52,292 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T01:40:52,295 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:52,298 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,42371,1732758050017, state=OPEN 2024-11-28T01:40:52,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:40:52,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:40:52,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:40:52,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:40:52,304 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:52,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T01:40:52,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,42371,1732758050017 in 322 msec 2024-11-28T01:40:52,323 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:40:52,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T01:40:52,323 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T01:40:52,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 558 msec 2024-11-28T01:40:52,350 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:52,351 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:52,381 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:52,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36313, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:52,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3040 sec 2024-11-28T01:40:52,412 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732758052412, completionTime=-1 2024-11-28T01:40:52,415 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T01:40:52,416 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T01:40:52,492 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T01:40:52,493 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732758112493 2024-11-28T01:40:52,493 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732758172493 2024-11-28T01:40:52,493 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 77 msec 2024-11-28T01:40:52,495 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45903,1732758049085-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:52,496 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45903,1732758049085-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:52,496 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45903,1732758049085-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:52,498 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9df3f25f5867:45903, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:52,498 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:52,499 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:52,507 DEBUG [master/9df3f25f5867:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T01:40:52,534 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.317sec 2024-11-28T01:40:52,536 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T01:40:52,537 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T01:40:52,538 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T01:40:52,539 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T01:40:52,539 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T01:40:52,540 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45903,1732758049085-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:40:52,540 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45903,1732758049085-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T01:40:52,549 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T01:40:52,550 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T01:40:52,551 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45903,1732758049085-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:40:52,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dfad15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:52,564 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T01:40:52,564 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T01:40:52,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:52,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:52,579 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6c304e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:52,595 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:52,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b5c665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:52,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:52,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:52,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9df3f25f5867,45903,1732758049085 2024-11-28T01:40:52,646 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=429, ProcessCount=11, AvailableMemoryMB=7910 2024-11-28T01:40:52,656 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:40:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,45903,1732758049085 2024-11-28T01:40:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@e3ff8cb 2024-11-28T01:40:52,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:40:52,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:40:52,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:40:52,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:40:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T01:40:52,692 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:40:52,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-28T01:40:52,695 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:52,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:40:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:40:52,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741835_1011 (size=1061) 2024-11-28T01:40:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:40:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:40:53,112 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:40:53,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741836_1012 (size=53) 2024-11-28T01:40:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:40:53,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:40:53,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 707ae842769f4709bfcd7372850df285, disabling compactions & flushes 2024-11-28T01:40:53,523 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. after waiting 0 ms 2024-11-28T01:40:53,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,523 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,523 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 707ae842769f4709bfcd7372850df285: Waiting for close lock at 1732758053523Disabling compacts and flushes for region at 1732758053523Disabling writes for close at 1732758053523Writing region close event to WAL at 1732758053523Closed at 1732758053523 2024-11-28T01:40:53,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:40:53,531 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732758053526"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758053526"}]},"ts":"1732758053526"} 2024-11-28T01:40:53,536 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:40:53,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:40:53,541 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758053538"}]},"ts":"1732758053538"} 2024-11-28T01:40:53,546 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T01:40:53,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, ASSIGN}] 2024-11-28T01:40:53,550 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, ASSIGN 2024-11-28T01:40:53,552 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, ASSIGN; state=OFFLINE, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=false 2024-11-28T01:40:53,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=707ae842769f4709bfcd7372850df285, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:53,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, ASSIGN because future has completed 2024-11-28T01:40:53,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:40:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:40:53,871 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,871 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:40:53,872 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,872 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:40:53,872 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,872 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,875 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,879 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:40:53,879 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 707ae842769f4709bfcd7372850df285 columnFamilyName A 2024-11-28T01:40:53,879 DEBUG [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:53,881 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.HStore(327): Store=707ae842769f4709bfcd7372850df285/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:40:53,881 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,883 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:40:53,884 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 707ae842769f4709bfcd7372850df285 columnFamilyName B 2024-11-28T01:40:53,884 DEBUG [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:53,885 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.HStore(327): Store=707ae842769f4709bfcd7372850df285/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:40:53,885 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,887 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:40:53,888 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 707ae842769f4709bfcd7372850df285 columnFamilyName C 2024-11-28T01:40:53,888 DEBUG [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:40:53,888 INFO [StoreOpener-707ae842769f4709bfcd7372850df285-1 {}] regionserver.HStore(327): Store=707ae842769f4709bfcd7372850df285/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:40:53,889 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,889 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,891 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,891 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,893 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,893 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,894 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:40:53,897 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,902 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:40:53,903 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 707ae842769f4709bfcd7372850df285; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68465314, jitterRate=0.020212680101394653}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:40:53,903 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:53,905 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 707ae842769f4709bfcd7372850df285: Running coprocessor pre-open hook at 1732758053872Writing region info on filesystem at 1732758053872Initializing all the Stores at 1732758053874 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758053874Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758053874Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758053874Cleaning up temporary data from old regions at 1732758053893 (+19 ms)Running coprocessor post-open hooks at 1732758053903 (+10 ms)Region opened successfully at 1732758053905 (+2 ms) 2024-11-28T01:40:53,907 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., pid=6, masterSystemTime=1732758053864 2024-11-28T01:40:53,911 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,911 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:53,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=707ae842769f4709bfcd7372850df285, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:53,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:40:53,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T01:40:53,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 in 209 msec 2024-11-28T01:40:53,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T01:40:53,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, ASSIGN in 375 msec 2024-11-28T01:40:53,929 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:40:53,929 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758053929"}]},"ts":"1732758053929"} 2024-11-28T01:40:53,932 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T01:40:53,934 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:40:53,938 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2510 sec 2024-11-28T01:40:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:40:54,854 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:40:54,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acda54d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,861 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328e9320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,865 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec4a323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,867 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,874 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:40:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,45903,1732758049085 2024-11-28T01:40:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@30e64719 2024-11-28T01:40:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:40:54,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:40:54,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4acbd0d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd90ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,893 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bb8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,897 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,900 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,910 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,913 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e412ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16775ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0c6470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@537574c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,924 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4153de0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,927 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc5ceca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46783332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,932 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e877dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,935 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35351cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:40:54,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:40:54,937 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:40:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:40:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:40:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c39882a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:40:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:40:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:40:54,939 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:40:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1ec368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:40:54,943 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,945 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,945 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:40:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,952 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,953 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-28T01:40:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:40:54,957 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T01:40:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:40:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:40:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,976 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,978 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,981 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:40:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,983 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:40:54,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,985 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:40:54,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:40:54,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:54,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:40:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T01:40:55,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:40:55,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:40:55,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:55,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:40:55,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:55,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:40:55,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:55,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T01:40:55,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:55,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758115200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758115208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758115213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1a3762a22fd7450b8176a14548e9a8e3 is 50, key is test_row_0/A:col10/1732758055080/Put/seqid=0 2024-11-28T01:40:55,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758115226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758115228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T01:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741837_1013 (size=12001) 2024-11-28T01:40:55,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1a3762a22fd7450b8176a14548e9a8e3 2024-11-28T01:40:55,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T01:40:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758115395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758115396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758115397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758115398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758115398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f63dbe6fdc334c9582712f41f03431df is 50, key is test_row_0/B:col10/1732758055080/Put/seqid=0 2024-11-28T01:40:55,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741838_1014 (size=12001) 2024-11-28T01:40:55,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T01:40:55,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:55,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T01:40:55,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758115604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758115613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758115613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758115623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758115623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T01:40:55,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:55,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T01:40:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f63dbe6fdc334c9582712f41f03431df 2024-11-28T01:40:55,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/b5d92fc43c5347008d669898688ebc4b is 50, key is test_row_0/C:col10/1732758055080/Put/seqid=0 2024-11-28T01:40:55,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758115924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758115924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758115924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758115933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758115943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:55,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741839_1015 (size=12001) 2024-11-28T01:40:55,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/b5d92fc43c5347008d669898688ebc4b 2024-11-28T01:40:55,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1a3762a22fd7450b8176a14548e9a8e3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a3762a22fd7450b8176a14548e9a8e3 2024-11-28T01:40:55,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T01:40:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:55,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:55,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a3762a22fd7450b8176a14548e9a8e3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T01:40:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:56,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f63dbe6fdc334c9582712f41f03431df as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f63dbe6fdc334c9582712f41f03431df 2024-11-28T01:40:56,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f63dbe6fdc334c9582712f41f03431df, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T01:40:56,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/b5d92fc43c5347008d669898688ebc4b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b5d92fc43c5347008d669898688ebc4b 2024-11-28T01:40:56,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b5d92fc43c5347008d669898688ebc4b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-28T01:40:56,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 707ae842769f4709bfcd7372850df285 in 959ms, sequenceid=14, compaction requested=false 2024-11-28T01:40:56,043 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T01:40:56,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T01:40:56,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T01:40:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:56,160 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:40:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:40:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:40:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:40:56,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:56,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/e7d76c4930de4172b3830e8e1e50dc2d is 50, key is test_row_0/A:col10/1732758055222/Put/seqid=0 2024-11-28T01:40:56,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741840_1016 (size=12001) 2024-11-28T01:40:56,225 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/e7d76c4930de4172b3830e8e1e50dc2d 2024-11-28T01:40:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4c8a684de3e64828a5d71a2fdc6c699b is 50, key is test_row_0/B:col10/1732758055222/Put/seqid=0 2024-11-28T01:40:56,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741841_1017 (size=12001) 2024-11-28T01:40:56,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4c8a684de3e64828a5d71a2fdc6c699b 2024-11-28T01:40:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/36c8f77911714f258e4852452ce1b504 is 50, key is test_row_0/C:col10/1732758055222/Put/seqid=0 2024-11-28T01:40:56,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741842_1018 (size=12001) 2024-11-28T01:40:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:56,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758116463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758116475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758116478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758116478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758116480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758116593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758116594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758116594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758116604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758116604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,740 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/36c8f77911714f258e4852452ce1b504 2024-11-28T01:40:56,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/e7d76c4930de4172b3830e8e1e50dc2d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/e7d76c4930de4172b3830e8e1e50dc2d 2024-11-28T01:40:56,783 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/e7d76c4930de4172b3830e8e1e50dc2d, entries=150, sequenceid=34, filesize=11.7 K 2024-11-28T01:40:56,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4c8a684de3e64828a5d71a2fdc6c699b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4c8a684de3e64828a5d71a2fdc6c699b 2024-11-28T01:40:56,806 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4c8a684de3e64828a5d71a2fdc6c699b, entries=150, sequenceid=34, filesize=11.7 K 2024-11-28T01:40:56,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/36c8f77911714f258e4852452ce1b504 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/36c8f77911714f258e4852452ce1b504 2024-11-28T01:40:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758116804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758116805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758116804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,824 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/36c8f77911714f258e4852452ce1b504, entries=150, sequenceid=34, filesize=11.7 K 2024-11-28T01:40:56,827 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 707ae842769f4709bfcd7372850df285 in 668ms, sequenceid=34, compaction requested=false 2024-11-28T01:40:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-28T01:40:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-28T01:40:56,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T01:40:56,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-11-28T01:40:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:56,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:40:56,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:40:56,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:56,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:40:56,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:56,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:40:56,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:56,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8910 sec 2024-11-28T01:40:56,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ce66cfb4b46f4f23aaf0f51c0138da3d is 50, key is test_row_0/A:col10/1732758056472/Put/seqid=0 2024-11-28T01:40:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741843_1019 (size=12001) 2024-11-28T01:40:56,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ce66cfb4b46f4f23aaf0f51c0138da3d 2024-11-28T01:40:56,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/57eef107197e41a5a84818a5ef1da2af is 50, key is test_row_0/B:col10/1732758056472/Put/seqid=0 2024-11-28T01:40:56,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758116937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758116941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:56,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741844_1020 (size=12001) 2024-11-28T01:40:56,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/57eef107197e41a5a84818a5ef1da2af 2024-11-28T01:40:57,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0bdecff9fe69418c944d6493723793fe is 50, key is test_row_0/C:col10/1732758056472/Put/seqid=0 2024-11-28T01:40:57,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741845_1021 (size=12001) 2024-11-28T01:40:57,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0bdecff9fe69418c944d6493723793fe 2024-11-28T01:40:57,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ce66cfb4b46f4f23aaf0f51c0138da3d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ce66cfb4b46f4f23aaf0f51c0138da3d 2024-11-28T01:40:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758117054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758117054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ce66cfb4b46f4f23aaf0f51c0138da3d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T01:40:57,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/57eef107197e41a5a84818a5ef1da2af as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/57eef107197e41a5a84818a5ef1da2af 2024-11-28T01:40:57,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/57eef107197e41a5a84818a5ef1da2af, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T01:40:57,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0bdecff9fe69418c944d6493723793fe as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0bdecff9fe69418c944d6493723793fe 2024-11-28T01:40:57,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0bdecff9fe69418c944d6493723793fe, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T01:40:57,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 707ae842769f4709bfcd7372850df285 in 270ms, sequenceid=48, compaction requested=true 2024-11-28T01:40:57,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:40:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T01:40:57,113 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:40:57,114 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:40:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-28T01:40:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T01:40:57,117 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:40:57,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:40:57,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:40:57,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:40:57,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:40:57,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:40:57,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:40:57,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:40:57,128 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:40:57,130 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:40:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:57,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:40:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:40:57,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:40:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:57,133 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:40:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:40:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:57,135 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:40:57,137 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:40:57,137 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:40:57,138 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,138 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,138 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a3762a22fd7450b8176a14548e9a8e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/e7d76c4930de4172b3830e8e1e50dc2d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ce66cfb4b46f4f23aaf0f51c0138da3d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=35.2 K 2024-11-28T01:40:57,138 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f63dbe6fdc334c9582712f41f03431df, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4c8a684de3e64828a5d71a2fdc6c699b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/57eef107197e41a5a84818a5ef1da2af] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=35.2 K 2024-11-28T01:40:57,140 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a3762a22fd7450b8176a14548e9a8e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732758055035 2024-11-28T01:40:57,140 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f63dbe6fdc334c9582712f41f03431df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732758055035 2024-11-28T01:40:57,141 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7d76c4930de4172b3830e8e1e50dc2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732758055185 2024-11-28T01:40:57,141 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c8a684de3e64828a5d71a2fdc6c699b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732758055185 2024-11-28T01:40:57,143 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce66cfb4b46f4f23aaf0f51c0138da3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732758056472 2024-11-28T01:40:57,144 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 57eef107197e41a5a84818a5ef1da2af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732758056472 2024-11-28T01:40:57,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/36aec1b97d244a5ebc25f8849748df1d is 50, key is test_row_0/A:col10/1732758057127/Put/seqid=0 2024-11-28T01:40:57,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741846_1022 (size=14341) 2024-11-28T01:40:57,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/36aec1b97d244a5ebc25f8849748df1d 2024-11-28T01:40:57,212 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#11 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:40:57,212 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:40:57,214 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/bf496e0df75e4b4ba536311bbea86f8f is 50, key is test_row_0/B:col10/1732758056472/Put/seqid=0 2024-11-28T01:40:57,214 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/63ef675393804f88a36a56469cb7c582 is 50, key is test_row_0/A:col10/1732758056472/Put/seqid=0 2024-11-28T01:40:57,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758117188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758117197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758117206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T01:40:57,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/738330cb98664ddaa45a43a5ba620253 is 50, key is test_row_0/B:col10/1732758057127/Put/seqid=0 2024-11-28T01:40:57,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741847_1023 (size=12104) 2024-11-28T01:40:57,266 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/63ef675393804f88a36a56469cb7c582 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/63ef675393804f88a36a56469cb7c582 2024-11-28T01:40:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758117263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:57,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741848_1024 (size=12104) 2024-11-28T01:40:57,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758117274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,292 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 63ef675393804f88a36a56469cb7c582(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:40:57,292 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:57,294 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/bf496e0df75e4b4ba536311bbea86f8f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/bf496e0df75e4b4ba536311bbea86f8f 2024-11-28T01:40:57,298 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758057110; duration=0sec 2024-11-28T01:40:57,298 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:40:57,299 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:40:57,299 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:40:57,307 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:40:57,308 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:40:57,308 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,308 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b5d92fc43c5347008d669898688ebc4b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/36c8f77911714f258e4852452ce1b504, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0bdecff9fe69418c944d6493723793fe] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=35.2 K 2024-11-28T01:40:57,310 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into bf496e0df75e4b4ba536311bbea86f8f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:40:57,310 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:57,311 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758057125; duration=0sec 2024-11-28T01:40:57,311 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:40:57,311 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:40:57,318 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d92fc43c5347008d669898688ebc4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732758055035 2024-11-28T01:40:57,319 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36c8f77911714f258e4852452ce1b504, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732758055185 2024-11-28T01:40:57,320 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bdecff9fe69418c944d6493723793fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732758056472 2024-11-28T01:40:57,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741849_1025 (size=12001) 2024-11-28T01:40:57,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758117323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758117323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758117333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,351 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:40:57,353 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/fb310e9761ab4a2b90441777bf31bfa6 is 50, key is test_row_0/C:col10/1732758056472/Put/seqid=0 2024-11-28T01:40:57,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741850_1026 (size=12104) 2024-11-28T01:40:57,390 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/fb310e9761ab4a2b90441777bf31bfa6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/fb310e9761ab4a2b90441777bf31bfa6 2024-11-28T01:40:57,406 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into fb310e9761ab4a2b90441777bf31bfa6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:40:57,406 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:57,406 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758057125; duration=0sec 2024-11-28T01:40:57,407 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:40:57,407 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:40:57,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T01:40:57,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758117533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758117542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758117543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758117583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758117593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/738330cb98664ddaa45a43a5ba620253 2024-11-28T01:40:57,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:57,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T01:40:57,753 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:40:57,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/11abecd61a5c4bfdbfae5882cce7a424 is 50, key is test_row_0/C:col10/1732758057127/Put/seqid=0 2024-11-28T01:40:57,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741851_1027 (size=12001) 2024-11-28T01:40:57,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758117843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758117854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758117855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:57,889 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T01:40:57,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:57,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:57,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:57,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758118103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758118112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/11abecd61a5c4bfdbfae5882cce7a424 2024-11-28T01:40:58,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/36aec1b97d244a5ebc25f8849748df1d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/36aec1b97d244a5ebc25f8849748df1d 2024-11-28T01:40:58,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/36aec1b97d244a5ebc25f8849748df1d, entries=200, sequenceid=67, filesize=14.0 K 2024-11-28T01:40:58,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/738330cb98664ddaa45a43a5ba620253 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/738330cb98664ddaa45a43a5ba620253 2024-11-28T01:40:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T01:40:58,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/738330cb98664ddaa45a43a5ba620253, entries=150, sequenceid=67, filesize=11.7 K 2024-11-28T01:40:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/11abecd61a5c4bfdbfae5882cce7a424 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/11abecd61a5c4bfdbfae5882cce7a424 2024-11-28T01:40:58,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/11abecd61a5c4bfdbfae5882cce7a424, entries=150, sequenceid=67, filesize=11.7 K 2024-11-28T01:40:58,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 707ae842769f4709bfcd7372850df285 in 1170ms, sequenceid=67, compaction requested=false 2024-11-28T01:40:58,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:58,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:40:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:40:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:40:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:40:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:58,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/b4de58042086417a92f08c5aa8ce59fc is 50, key is test_row_0/A:col10/1732758057187/Put/seqid=0 2024-11-28T01:40:58,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741852_1028 (size=14341) 2024-11-28T01:40:58,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/b4de58042086417a92f08c5aa8ce59fc 2024-11-28T01:40:58,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/86bff3ebe90f46d09bc8a2fd5109c18b is 50, key is test_row_0/B:col10/1732758057187/Put/seqid=0 2024-11-28T01:40:58,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758118476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758118477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758118477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741853_1029 (size=12001) 2024-11-28T01:40:58,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758118582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758118603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758118603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:58,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:58,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758118803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758118813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758118813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:58,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:58,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:58,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:58,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:58,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/86bff3ebe90f46d09bc8a2fd5109c18b 2024-11-28T01:40:58,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a043dc5b2c7d4696817aad2c5e1d43b1 is 50, key is test_row_0/C:col10/1732758057187/Put/seqid=0 2024-11-28T01:40:58,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741854_1030 (size=12001) 2024-11-28T01:40:59,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758119123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758119124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758119124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758119124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,129 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:40:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758119133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,140 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_0' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:40:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:59,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:59,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T01:40:59,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:59,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a043dc5b2c7d4696817aad2c5e1d43b1 2024-11-28T01:40:59,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/b4de58042086417a92f08c5aa8ce59fc as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/b4de58042086417a92f08c5aa8ce59fc 2024-11-28T01:40:59,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/b4de58042086417a92f08c5aa8ce59fc, entries=200, sequenceid=84, filesize=14.0 K 2024-11-28T01:40:59,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/86bff3ebe90f46d09bc8a2fd5109c18b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/86bff3ebe90f46d09bc8a2fd5109c18b 2024-11-28T01:40:59,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:59,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:59,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/86bff3ebe90f46d09bc8a2fd5109c18b, entries=150, sequenceid=84, filesize=11.7 K 2024-11-28T01:40:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:40:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a043dc5b2c7d4696817aad2c5e1d43b1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a043dc5b2c7d4696817aad2c5e1d43b1 2024-11-28T01:40:59,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a043dc5b2c7d4696817aad2c5e1d43b1, entries=150, sequenceid=84, filesize=11.7 K 2024-11-28T01:40:59,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 707ae842769f4709bfcd7372850df285 in 1166ms, sequenceid=84, compaction requested=true 2024-11-28T01:40:59,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:40:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:40:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:40:59,525 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:40:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:40:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:40:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:40:59,525 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:40:59,528 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:40:59,530 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:40:59,530 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,530 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/63ef675393804f88a36a56469cb7c582, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/36aec1b97d244a5ebc25f8849748df1d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/b4de58042086417a92f08c5aa8ce59fc] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=39.8 K 2024-11-28T01:40:59,531 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:40:59,532 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 63ef675393804f88a36a56469cb7c582, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732758056472 2024-11-28T01:40:59,532 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:40:59,532 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,532 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/bf496e0df75e4b4ba536311bbea86f8f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/738330cb98664ddaa45a43a5ba620253, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/86bff3ebe90f46d09bc8a2fd5109c18b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=35.3 K 2024-11-28T01:40:59,533 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 36aec1b97d244a5ebc25f8849748df1d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732758056891 2024-11-28T01:40:59,534 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf496e0df75e4b4ba536311bbea86f8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732758056472 2024-11-28T01:40:59,534 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b4de58042086417a92f08c5aa8ce59fc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732758057147 2024-11-28T01:40:59,536 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738330cb98664ddaa45a43a5ba620253, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732758056932 2024-11-28T01:40:59,538 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86bff3ebe90f46d09bc8a2fd5109c18b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732758057147 2024-11-28T01:40:59,570 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:40:59,572 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d693de301e40487fa787d499fd4c31f6 is 50, key is test_row_0/B:col10/1732758057187/Put/seqid=0 2024-11-28T01:40:59,583 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:40:59,589 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/708feaa7657a4d939de04c27ee5333fd is 50, key is test_row_0/A:col10/1732758057187/Put/seqid=0 2024-11-28T01:40:59,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741855_1031 (size=12207) 2024-11-28T01:40:59,630 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T01:40:59,630 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d693de301e40487fa787d499fd4c31f6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d693de301e40487fa787d499fd4c31f6 2024-11-28T01:40:59,631 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T01:40:59,633 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T01:40:59,633 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T01:40:59,633 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T01:40:59,634 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T01:40:59,635 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T01:40:59,635 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T01:40:59,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T01:40:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741856_1032 (size=12207) 2024-11-28T01:40:59,680 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:40:59,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:40:59,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:40:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:40:59,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:40:59,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:40:59,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:40:59,693 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into d693de301e40487fa787d499fd4c31f6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:40:59,693 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:59,693 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758059525; duration=0sec 2024-11-28T01:40:59,693 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:40:59,694 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:40:59,694 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:40:59,699 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:40:59,699 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:40:59,699 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:40:59,699 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/fb310e9761ab4a2b90441777bf31bfa6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/11abecd61a5c4bfdbfae5882cce7a424, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a043dc5b2c7d4696817aad2c5e1d43b1] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=35.3 K 2024-11-28T01:40:59,701 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb310e9761ab4a2b90441777bf31bfa6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732758056472 2024-11-28T01:40:59,706 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11abecd61a5c4bfdbfae5882cce7a424, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732758056932 2024-11-28T01:40:59,707 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a043dc5b2c7d4696817aad2c5e1d43b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732758057147 2024-11-28T01:40:59,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/9d51503ae92242c79e13eec3d7566ed2 is 50, key is test_row_0/A:col10/1732758058446/Put/seqid=0 2024-11-28T01:40:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758119730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,748 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:40:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/06b6bb5258c44a459bb0aa1d13eb5266 is 50, key is test_row_0/C:col10/1732758057187/Put/seqid=0 2024-11-28T01:40:59,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758119746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758119748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741857_1033 (size=12001) 2024-11-28T01:40:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741858_1034 (size=12207) 2024-11-28T01:40:59,796 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/06b6bb5258c44a459bb0aa1d13eb5266 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/06b6bb5258c44a459bb0aa1d13eb5266 2024-11-28T01:40:59,812 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into 06b6bb5258c44a459bb0aa1d13eb5266(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:40:59,812 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:40:59,813 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758059525; duration=0sec 2024-11-28T01:40:59,813 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:40:59,813 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:40:59,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758119853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758119863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:40:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:40:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758119864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:40:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:40:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758120063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758120073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758120073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,117 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/708feaa7657a4d939de04c27ee5333fd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/708feaa7657a4d939de04c27ee5333fd 2024-11-28T01:41:00,146 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 708feaa7657a4d939de04c27ee5333fd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:00,146 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:00,146 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758059524; duration=0sec 2024-11-28T01:41:00,146 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:00,146 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:00,161 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/9d51503ae92242c79e13eec3d7566ed2 2024-11-28T01:41:00,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/ca40c2170aae4b618776fa865d63b5cf is 50, key is test_row_0/B:col10/1732758058446/Put/seqid=0 2024-11-28T01:41:00,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741859_1035 (size=12001) 2024-11-28T01:41:00,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/ca40c2170aae4b618776fa865d63b5cf 2024-11-28T01:41:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/4a72e13870074460bc210727affc7176 is 50, key is test_row_0/C:col10/1732758058446/Put/seqid=0 2024-11-28T01:41:00,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741860_1036 (size=12001) 2024-11-28T01:41:00,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758120373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758120383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758120392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,692 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/4a72e13870074460bc210727affc7176 2024-11-28T01:41:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/9d51503ae92242c79e13eec3d7566ed2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d51503ae92242c79e13eec3d7566ed2 2024-11-28T01:41:00,713 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d51503ae92242c79e13eec3d7566ed2, entries=150, sequenceid=104, filesize=11.7 K 2024-11-28T01:41:00,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/ca40c2170aae4b618776fa865d63b5cf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ca40c2170aae4b618776fa865d63b5cf 2024-11-28T01:41:00,733 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ca40c2170aae4b618776fa865d63b5cf, entries=150, sequenceid=104, filesize=11.7 K 2024-11-28T01:41:00,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/4a72e13870074460bc210727affc7176 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/4a72e13870074460bc210727affc7176 2024-11-28T01:41:00,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/4a72e13870074460bc210727affc7176, entries=150, sequenceid=104, filesize=11.7 K 2024-11-28T01:41:00,747 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 707ae842769f4709bfcd7372850df285 in 1068ms, sequenceid=104, compaction requested=false 2024-11-28T01:41:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-28T01:41:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-28T01:41:00,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-28T01:41:00,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6310 sec 2024-11-28T01:41:00,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 3.6420 sec 2024-11-28T01:41:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:00,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:41:00,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:00,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:00,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/744b45ad9efd4f21b383c398c88ee63e is 50, key is test_row_0/A:col10/1732758059708/Put/seqid=0 2024-11-28T01:41:00,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741861_1037 (size=14341) 2024-11-28T01:41:00,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/744b45ad9efd4f21b383c398c88ee63e 2024-11-28T01:41:00,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/dcc0eec9f71e47f797924a1b2621228b is 50, key is test_row_0/B:col10/1732758059708/Put/seqid=0 2024-11-28T01:41:00,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758120941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758120945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758120946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:00,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741862_1038 (size=12001) 2024-11-28T01:41:00,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/dcc0eec9f71e47f797924a1b2621228b 2024-11-28T01:41:00,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/bab6ed42d5d74b7aaffeba8c2f32546b is 50, key is test_row_0/C:col10/1732758059708/Put/seqid=0 2024-11-28T01:41:01,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741863_1039 (size=12001) 2024-11-28T01:41:01,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/bab6ed42d5d74b7aaffeba8c2f32546b 2024-11-28T01:41:01,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/744b45ad9efd4f21b383c398c88ee63e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/744b45ad9efd4f21b383c398c88ee63e 2024-11-28T01:41:01,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/744b45ad9efd4f21b383c398c88ee63e, entries=200, sequenceid=120, filesize=14.0 K 2024-11-28T01:41:01,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/dcc0eec9f71e47f797924a1b2621228b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/dcc0eec9f71e47f797924a1b2621228b 2024-11-28T01:41:01,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/dcc0eec9f71e47f797924a1b2621228b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-28T01:41:01,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/bab6ed42d5d74b7aaffeba8c2f32546b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bab6ed42d5d74b7aaffeba8c2f32546b 2024-11-28T01:41:01,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bab6ed42d5d74b7aaffeba8c2f32546b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-28T01:41:01,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 707ae842769f4709bfcd7372850df285 in 167ms, sequenceid=120, compaction requested=true 2024-11-28T01:41:01,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:01,057 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:01,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:01,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:01,058 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:01,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:01,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:01,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:01,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:01,062 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:01,062 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:01,062 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,062 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/708feaa7657a4d939de04c27ee5333fd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d51503ae92242c79e13eec3d7566ed2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/744b45ad9efd4f21b383c398c88ee63e] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=37.6 K 2024-11-28T01:41:01,063 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:01,063 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708feaa7657a4d939de04c27ee5333fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732758057147 2024-11-28T01:41:01,063 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:01,063 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,063 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d693de301e40487fa787d499fd4c31f6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ca40c2170aae4b618776fa865d63b5cf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/dcc0eec9f71e47f797924a1b2621228b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=35.4 K 2024-11-28T01:41:01,064 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d51503ae92242c79e13eec3d7566ed2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732758058446 2024-11-28T01:41:01,064 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d693de301e40487fa787d499fd4c31f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732758057147 2024-11-28T01:41:01,065 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744b45ad9efd4f21b383c398c88ee63e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758059684 2024-11-28T01:41:01,065 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ca40c2170aae4b618776fa865d63b5cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732758058446 2024-11-28T01:41:01,067 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc0eec9f71e47f797924a1b2621228b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758059684 2024-11-28T01:41:01,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758060938, max timestamp=1732758061058Num uniques 859; 2024-11-28T01:41:01,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:01,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758060938, max timestamp=1732758061058Num uniques 859; 2024-11-28T01:41:01,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:01,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=62.62 KB heapSize=174.33 KB 2024-11-28T01:41:01,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:01,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758060938, max timestamp=1732758061058Num uniques 859; 2024-11-28T01:41:01,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:01,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:01,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:01,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:01,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:01,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:01,090 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:01,091 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f2ac74457b2f4b17ad169b5434ee9cfe is 50, key is test_row_0/B:col10/1732758059708/Put/seqid=0 2024-11-28T01:41:01,101 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:01,102 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/c115e3f3034c4e6b9072b49054d0f117 is 50, key is test_row_0/A:col10/1732758059708/Put/seqid=0 2024-11-28T01:41:01,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/f0caef03c28d4d519fb41a78c007fad2 is 50, key is test_row_0/A:col10/1732758060939/Put/seqid=0 2024-11-28T01:41:01,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741864_1040 (size=12309) 2024-11-28T01:41:01,171 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f2ac74457b2f4b17ad169b5434ee9cfe as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f2ac74457b2f4b17ad169b5434ee9cfe 2024-11-28T01:41:01,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741865_1041 (size=12309) 2024-11-28T01:41:01,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741866_1042 (size=12151) 2024-11-28T01:41:01,184 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/c115e3f3034c4e6b9072b49054d0f117 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c115e3f3034c4e6b9072b49054d0f117 2024-11-28T01:41:01,185 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into f2ac74457b2f4b17ad169b5434ee9cfe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:01,185 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:01,185 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758061058; duration=0sec 2024-11-28T01:41:01,186 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:01,186 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:01,186 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:01,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758061119, max timestamp=1732758061166Num uniques 859; 2024-11-28T01:41:01,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:01,190 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:01,190 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:01,190 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,190 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/06b6bb5258c44a459bb0aa1d13eb5266, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/4a72e13870074460bc210727affc7176, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bab6ed42d5d74b7aaffeba8c2f32546b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=35.4 K 2024-11-28T01:41:01,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758061119, max timestamp=1732758061154Num uniques 859; 2024-11-28T01:41:01,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758061119, max timestamp=1732758061170Num uniques 859; 2024-11-28T01:41:01,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:01,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:01,191 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b6bb5258c44a459bb0aa1d13eb5266, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732758057147 2024-11-28T01:41:01,192 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a72e13870074460bc210727affc7176, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732758058446 2024-11-28T01:41:01,193 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting bab6ed42d5d74b7aaffeba8c2f32546b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758059684 2024-11-28T01:41:01,196 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into c115e3f3034c4e6b9072b49054d0f117(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:01,197 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:01,197 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758061057; duration=0sec 2024-11-28T01:41:01,197 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:01,197 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:01,235 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:01,238 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/1fdeb1377646414cae643d159f63aad0 is 50, key is test_row_0/C:col10/1732758059708/Put/seqid=0 2024-11-28T01:41:01,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741867_1043 (size=12309) 2024-11-28T01:41:01,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758061194, max timestamp=1732758061248Num uniques 1018; 2024-11-28T01:41:01,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-28T01:41:01,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758061194, max timestamp=1732758061250Num uniques 1018; 2024-11-28T01:41:01,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-28T01:41:01,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758061194, max timestamp=1732758061248Num uniques 1018; 2024-11-28T01:41:01,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-28T01:41:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T01:41:01,294 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:01,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-28T01:41:01,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T01:41:01,298 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:01,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:01,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758121311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758121314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732758061297, max timestamp=1732758061314Num uniques 1077; 2024-11-28T01:41:01,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-28T01:41:01,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732758061297, max timestamp=1732758061314Num uniques 1077; 2024-11-28T01:41:01,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-28T01:41:01,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732758061297, max timestamp=1732758061314Num uniques 1077; 2024-11-28T01:41:01,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-28T01:41:01,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732758061358, max timestamp=1732758061387Num uniques 1086; 2024-11-28T01:41:01,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732758061358, max timestamp=1732758061387Num uniques 1086; 2024-11-28T01:41:01,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-28T01:41:01,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732758061358, max timestamp=1732758061387Num uniques 1086; 2024-11-28T01:41:01,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-28T01:41:01,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-28T01:41:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T01:41:01,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732758061440, max timestamp=1732758061458Num uniques 1195; 2024-11-28T01:41:01,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-11-28T01:41:01,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732758061440, max timestamp=1732758061460Num uniques 1195; 2024-11-28T01:41:01,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-11-28T01:41:01,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732758061440, max timestamp=1732758061460Num uniques 1195; 2024-11-28T01:41:01,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-11-28T01:41:01,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16204, totalHeapSize=37056, min timestamp=1732758061493, max timestamp=1732758061520Num uniques 1104; 2024-11-28T01:41:01,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1732758061493, max timestamp=1732758061520Num uniques 1104; 2024-11-28T01:41:01,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1732758061493, max timestamp=1732758061520Num uniques 1104; 2024-11-28T01:41:01,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50554, new segment data size=16204, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-11-28T01:41:01,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-11-28T01:41:01,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-11-28T01:41:01,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/f0caef03c28d4d519fb41a78c007fad2 2024-11-28T01:41:01,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4cda20d8e54a4d61a311a22b4b190739 is 50, key is test_row_0/B:col10/1732758060939/Put/seqid=0 2024-11-28T01:41:01,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:01,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758121597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758121598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:01,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758121599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T01:41:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758121611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758121611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741868_1044 (size=12151) 2024-11-28T01:41:01,670 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/1fdeb1377646414cae643d159f63aad0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/1fdeb1377646414cae643d159f63aad0 2024-11-28T01:41:01,681 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into 1fdeb1377646414cae643d159f63aad0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:01,681 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:01,681 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758061060; duration=0sec 2024-11-28T01:41:01,682 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:01,682 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:01,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758121723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758121723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758121728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758121728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758121735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T01:41:01,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:01,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758121933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758121933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758121942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758121943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758121943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4cda20d8e54a4d61a311a22b4b190739 2024-11-28T01:41:02,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/e9d945b974724d3aa58482a0d5356068 is 50, key is test_row_0/C:col10/1732758060939/Put/seqid=0 2024-11-28T01:41:02,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741869_1045 (size=12151) 2024-11-28T01:41:02,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:02,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:02,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758122242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758122243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758122252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758122252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758122253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T01:41:02,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/e9d945b974724d3aa58482a0d5356068 2024-11-28T01:41:02,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/f0caef03c28d4d519fb41a78c007fad2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/f0caef03c28d4d519fb41a78c007fad2 2024-11-28T01:41:02,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/f0caef03c28d4d519fb41a78c007fad2, entries=150, sequenceid=141, filesize=11.9 K 2024-11-28T01:41:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4cda20d8e54a4d61a311a22b4b190739 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4cda20d8e54a4d61a311a22b4b190739 2024-11-28T01:41:02,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4cda20d8e54a4d61a311a22b4b190739, entries=150, sequenceid=141, filesize=11.9 K 2024-11-28T01:41:02,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/e9d945b974724d3aa58482a0d5356068 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e9d945b974724d3aa58482a0d5356068 2024-11-28T01:41:02,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e9d945b974724d3aa58482a0d5356068, entries=150, sequenceid=141, filesize=11.9 K 2024-11-28T01:41:02,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~81.42 KB/83376, currentSize=161.02 KB/164880 for 707ae842769f4709bfcd7372850df285 in 1440ms, sequenceid=141, compaction requested=false 2024-11-28T01:41:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:02,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T01:41:02,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,546 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=161.02 KB heapSize=446.39 KB 2024-11-28T01:41:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1117bd21a9a84eaf9b1b1bd6c2b35c07 is 50, key is test_row_0/A:col10/1732758061594/Put/seqid=0 2024-11-28T01:41:02,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741870_1046 (size=12301) 2024-11-28T01:41:02,596 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1117bd21a9a84eaf9b1b1bd6c2b35c07 2024-11-28T01:41:02,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f5512650135543dba9db35448de7ef0b is 50, key is test_row_0/B:col10/1732758061594/Put/seqid=0 2024-11-28T01:41:02,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741871_1047 (size=12301) 2024-11-28T01:41:02,654 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f5512650135543dba9db35448de7ef0b 2024-11-28T01:41:02,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/b64e33236c8e4034917810d12318d0fb is 50, key is test_row_0/C:col10/1732758061594/Put/seqid=0 2024-11-28T01:41:02,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741872_1048 (size=12301) 2024-11-28T01:41:02,690 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/b64e33236c8e4034917810d12318d0fb 2024-11-28T01:41:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1117bd21a9a84eaf9b1b1bd6c2b35c07 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1117bd21a9a84eaf9b1b1bd6c2b35c07 2024-11-28T01:41:02,727 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1117bd21a9a84eaf9b1b1bd6c2b35c07, entries=150, sequenceid=267, filesize=12.0 K 2024-11-28T01:41:02,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f5512650135543dba9db35448de7ef0b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f5512650135543dba9db35448de7ef0b 2024-11-28T01:41:02,738 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f5512650135543dba9db35448de7ef0b, entries=150, sequenceid=267, filesize=12.0 K 2024-11-28T01:41:02,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/b64e33236c8e4034917810d12318d0fb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b64e33236c8e4034917810d12318d0fb 2024-11-28T01:41:02,751 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b64e33236c8e4034917810d12318d0fb, entries=150, sequenceid=267, filesize=12.0 K 2024-11-28T01:41:02,753 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~446.34 KB/457056, currentSize=0 B/0 for 707ae842769f4709bfcd7372850df285 in 208ms, sequenceid=267, compaction requested=true 2024-11-28T01:41:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-28T01:41:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-28T01:41:02,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T01:41:02,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-28T01:41:02,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4690 sec 2024-11-28T01:41:02,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:02,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:02,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:02,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:02,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:02,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ae2ca65929a54bc192960f4bed94e823 is 50, key is test_row_0/A:col10/1732758062763/Put/seqid=0 2024-11-28T01:41:02,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741873_1049 (size=14741) 2024-11-28T01:41:02,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ae2ca65929a54bc192960f4bed94e823 2024-11-28T01:41:02,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7b9724134dc7409ea3bc4781255371ec is 50, key is test_row_0/B:col10/1732758062763/Put/seqid=0 2024-11-28T01:41:02,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758062812, max timestamp=1732758062823Num uniques 859; 2024-11-28T01:41:02,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758062812, max timestamp=1732758062823Num uniques 859; 2024-11-28T01:41:02,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:02,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:02,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758062812, max timestamp=1732758062823Num uniques 859; 2024-11-28T01:41:02,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:02,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741874_1050 (size=12301) 2024-11-28T01:41:02,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7b9724134dc7409ea3bc4781255371ec 2024-11-28T01:41:02,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7bc04cfd2cb74045ba52369721463ede is 50, key is test_row_0/C:col10/1732758062763/Put/seqid=0 2024-11-28T01:41:02,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758122860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758122864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758122860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758122878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758122879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741875_1051 (size=12301) 2024-11-28T01:41:02,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758122982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758122982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758122993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758122993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758122993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:03,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758123193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:03,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758123193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:03,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758123202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:03,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:03,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758123203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:03,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758123203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:03,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:03,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7bc04cfd2cb74045ba52369721463ede 2024-11-28T01:41:03,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ae2ca65929a54bc192960f4bed94e823 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ae2ca65929a54bc192960f4bed94e823 2024-11-28T01:41:03,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ae2ca65929a54bc192960f4bed94e823, entries=200, sequenceid=277, filesize=14.4 K 2024-11-28T01:41:03,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7b9724134dc7409ea3bc4781255371ec as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7b9724134dc7409ea3bc4781255371ec 2024-11-28T01:41:03,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7b9724134dc7409ea3bc4781255371ec, entries=150, sequenceid=277, filesize=12.0 K 2024-11-28T01:41:03,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7bc04cfd2cb74045ba52369721463ede as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7bc04cfd2cb74045ba52369721463ede 2024-11-28T01:41:03,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7bc04cfd2cb74045ba52369721463ede, entries=150, sequenceid=277, filesize=12.0 K 2024-11-28T01:41:03,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 707ae842769f4709bfcd7372850df285 in 565ms, sequenceid=277, compaction requested=true 2024-11-28T01:41:03,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:03,339 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:03,339 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:03,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:03,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:03,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:03,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:03,341 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51502 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:03,341 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:03,341 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,341 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c115e3f3034c4e6b9072b49054d0f117, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/f0caef03c28d4d519fb41a78c007fad2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1117bd21a9a84eaf9b1b1bd6c2b35c07, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ae2ca65929a54bc192960f4bed94e823] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=50.3 K 2024-11-28T01:41:03,342 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:03,342 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:03,342 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,342 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f2ac74457b2f4b17ad169b5434ee9cfe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4cda20d8e54a4d61a311a22b4b190739, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f5512650135543dba9db35448de7ef0b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7b9724134dc7409ea3bc4781255371ec] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=47.9 K 2024-11-28T01:41:03,343 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c115e3f3034c4e6b9072b49054d0f117, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758059684 2024-11-28T01:41:03,343 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f2ac74457b2f4b17ad169b5434ee9cfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758059684 2024-11-28T01:41:03,343 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0caef03c28d4d519fb41a78c007fad2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732758060939 2024-11-28T01:41:03,344 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1117bd21a9a84eaf9b1b1bd6c2b35c07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732758061546 2024-11-28T01:41:03,344 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cda20d8e54a4d61a311a22b4b190739, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732758060939 2024-11-28T01:41:03,345 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae2ca65929a54bc192960f4bed94e823, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732758062756 2024-11-28T01:41:03,345 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f5512650135543dba9db35448de7ef0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732758061546 2024-11-28T01:41:03,346 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b9724134dc7409ea3bc4781255371ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732758062756 2024-11-28T01:41:03,370 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:03,371 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/de2bb5c72f8d460da4387f020a5605cd is 50, key is test_row_0/A:col10/1732758062763/Put/seqid=0 2024-11-28T01:41:03,376 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:03,377 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/91914a43f7c9432dac65590c786e0153 is 50, key is test_row_0/B:col10/1732758062763/Put/seqid=0 2024-11-28T01:41:03,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741876_1052 (size=12745) 2024-11-28T01:41:03,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741877_1053 (size=12745) 2024-11-28T01:41:03,408 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/91914a43f7c9432dac65590c786e0153 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/91914a43f7c9432dac65590c786e0153 2024-11-28T01:41:03,418 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into 91914a43f7c9432dac65590c786e0153(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:03,418 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:03,418 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=12, startTime=1732758063339; duration=0sec 2024-11-28T01:41:03,418 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:03,418 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:03,419 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:03,421 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:03,421 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:03,421 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,421 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/1fdeb1377646414cae643d159f63aad0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e9d945b974724d3aa58482a0d5356068, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b64e33236c8e4034917810d12318d0fb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7bc04cfd2cb74045ba52369721463ede] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=47.9 K 2024-11-28T01:41:03,422 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fdeb1377646414cae643d159f63aad0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758059684 2024-11-28T01:41:03,422 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d945b974724d3aa58482a0d5356068, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732758060939 2024-11-28T01:41:03,423 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b64e33236c8e4034917810d12318d0fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732758061546 2024-11-28T01:41:03,424 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bc04cfd2cb74045ba52369721463ede, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732758062756 2024-11-28T01:41:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T01:41:03,443 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:03,444 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-28T01:41:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T01:41:03,448 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:03,450 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:03,450 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/031278f9365e4b3ea610501083d3600c is 50, key is test_row_0/C:col10/1732758062763/Put/seqid=0 2024-11-28T01:41:03,451 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:03,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:03,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741878_1054 (size=12745) 2024-11-28T01:41:03,480 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/031278f9365e4b3ea610501083d3600c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/031278f9365e4b3ea610501083d3600c 2024-11-28T01:41:03,489 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into 031278f9365e4b3ea610501083d3600c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:03,489 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:03,489 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=12, startTime=1732758063341; duration=0sec 2024-11-28T01:41:03,489 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:03,489 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:03,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-28T01:41:03,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758062859, max timestamp=1732758063526Num uniques 1068; 2024-11-28T01:41:03,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T01:41:03,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758062859, max timestamp=1732758063526Num uniques 1068; 2024-11-28T01:41:03,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T01:41:03,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758062859, max timestamp=1732758063526Num uniques 1068; 2024-11-28T01:41:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:03,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-28T01:41:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:03,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:03,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/3665f894bc1d48cd89ad94d8e6690fb6 is 50, key is test_row_0/A:col10/1732758062875/Put/seqid=0 2024-11-28T01:41:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T01:41:03,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741879_1055 (size=12301) 2024-11-28T01:41:03,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T01:41:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758063560, max timestamp=1732758063628Num uniques 859; 2024-11-28T01:41:03,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:03,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758063560, max timestamp=1732758063628Num uniques 859; 2024-11-28T01:41:03,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758063560, max timestamp=1732758063628Num uniques 859; 2024-11-28T01:41:03,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:03,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:03,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758063664, max timestamp=1732758063687Num uniques 1018; 2024-11-28T01:41:03,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758063662, max timestamp=1732758063687Num uniques 1018; 2024-11-28T01:41:03,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-28T01:41:03,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-28T01:41:03,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758063664, max timestamp=1732758063687Num uniques 1018; 2024-11-28T01:41:03,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-28T01:41:03,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732758063727, max timestamp=1732758063740Num uniques 1027; 2024-11-28T01:41:03,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732758063727, max timestamp=1732758063740Num uniques 1077; 2024-11-28T01:41:03,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T01:41:03,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-28T01:41:03,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758063727, max timestamp=1732758063740Num uniques 1009; 2024-11-28T01:41:03,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-28T01:41:03,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T01:41:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T01:41:03,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732758063776, max timestamp=1732758063791Num uniques 1036; 2024-11-28T01:41:03,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T01:41:03,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732758063776, max timestamp=1732758063791Num uniques 1036; 2024-11-28T01:41:03,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732758063776, max timestamp=1732758063791Num uniques 1009; 2024-11-28T01:41:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-28T01:41:03,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13096, suffix heap size=132904,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=286 2024-11-28T01:41:03,799 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/de2bb5c72f8d460da4387f020a5605cd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/de2bb5c72f8d460da4387f020a5605cd 2024-11-28T01:41:03,810 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into de2bb5c72f8d460da4387f020a5605cd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:03,810 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:03,810 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=12, startTime=1732758063339; duration=0sec 2024-11-28T01:41:03,811 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:03,811 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:03,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732758063828, max timestamp=1732758063852Num uniques 1145; 2024-11-28T01:41:03,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-28T01:41:03,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732758063837, max timestamp=1732758063854Num uniques 1145; 2024-11-28T01:41:03,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-28T01:41:03,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758063837, max timestamp=1732758063852Num uniques 1145; 2024-11-28T01:41:03,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=6870, suffix heap size=147048,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=150 2024-11-28T01:41:03,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T01:41:03,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:03,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:03,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:03,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732758063888, max timestamp=1732758063917Num uniques 1154; 2024-11-28T01:41:03,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-11-28T01:41:03,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732758063888, max timestamp=1732758063917Num uniques 1054; 2024-11-28T01:41:03,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=13920, suffix heap size=137584,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=304 2024-11-28T01:41:03,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732758063901, max timestamp=1732758063920Num uniques 1009; 2024-11-28T01:41:03,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-11-28T01:41:03,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1732758063928, max timestamp=1732758063965Num uniques 1063; 2024-11-28T01:41:03,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=16618, suffix heap size=138520,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=363 2024-11-28T01:41:03,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1732758063928, max timestamp=1732758063965Num uniques 1163; 2024-11-28T01:41:03,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=16618, suffix heap size=148920,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=363 2024-11-28T01:41:03,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=363, cellSize=16618, totalHeapSize=37992, min timestamp=1732758063928, max timestamp=1732758063965Num uniques 1163; 2024-11-28T01:41:03,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=16618, suffix heap size=148920,new segment heap size=37992 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=363 2024-11-28T01:41:03,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/3665f894bc1d48cd89ad94d8e6690fb6 2024-11-28T01:41:04,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/997e283f1dba4c0f81ffc07b570f63d7 is 50, key is test_row_0/B:col10/1732758062875/Put/seqid=0 2024-11-28T01:41:04,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758124011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758124013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758124014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758124016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758124019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741880_1056 (size=12301) 2024-11-28T01:41:04,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T01:41:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:04,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T01:41:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758124132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758124133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758124133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758124133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758124134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T01:41:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:04,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758124352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758124353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758124353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758124353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758124353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T01:41:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:04,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:04,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/997e283f1dba4c0f81ffc07b570f63d7 2024-11-28T01:41:04,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/5585104e81344c548872da8d261565cf is 50, key is test_row_0/C:col10/1732758062875/Put/seqid=0 2024-11-28T01:41:04,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741881_1057 (size=12301) 2024-11-28T01:41:04,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/5585104e81344c548872da8d261565cf 2024-11-28T01:41:04,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/3665f894bc1d48cd89ad94d8e6690fb6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/3665f894bc1d48cd89ad94d8e6690fb6 2024-11-28T01:41:04,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/3665f894bc1d48cd89ad94d8e6690fb6, entries=150, sequenceid=318, filesize=12.0 K 2024-11-28T01:41:04,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/997e283f1dba4c0f81ffc07b570f63d7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/997e283f1dba4c0f81ffc07b570f63d7 2024-11-28T01:41:04,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/997e283f1dba4c0f81ffc07b570f63d7, entries=150, sequenceid=318, filesize=12.0 K 2024-11-28T01:41:04,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/5585104e81344c548872da8d261565cf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/5585104e81344c548872da8d261565cf 2024-11-28T01:41:04,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/5585104e81344c548872da8d261565cf, entries=150, sequenceid=318, filesize=12.0 K 2024-11-28T01:41:04,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.75 KB/33532, heapSize ~83.44 KB/85440, currentSize=161.02 KB/164880 for 707ae842769f4709bfcd7372850df285 in 975ms, sequenceid=318, compaction requested=false 2024-11-28T01:41:04,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:04,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T01:41:04,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:04,535 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=161.02 KB heapSize=445.55 KB 2024-11-28T01:41:04,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:04,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/72f455b93e354d3c8e4735ff895bbf20 is 50, key is test_row_0/A:col10/1732758064016/Put/seqid=0 2024-11-28T01:41:04,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741882_1058 (size=12301) 2024-11-28T01:41:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T01:41:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:04,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:04,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758124668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758124670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758124672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758124668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758124677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758124782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758124783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758124783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758124783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,951 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/72f455b93e354d3c8e4735ff895bbf20 2024-11-28T01:41:04,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/1ca4c6a5b4994b6c87ae0657fb26d518 is 50, key is test_row_0/B:col10/1732758064016/Put/seqid=0 2024-11-28T01:41:04,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741883_1059 (size=12301) 2024-11-28T01:41:04,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/1ca4c6a5b4994b6c87ae0657fb26d518 2024-11-28T01:41:04,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758124993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/96e58dc3e9ff4909aafebe640e63e6a5 is 50, key is test_row_0/C:col10/1732758064016/Put/seqid=0 2024-11-28T01:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758124993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:04,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758124995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758124995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741884_1060 (size=12301) 2024-11-28T01:41:05,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/96e58dc3e9ff4909aafebe640e63e6a5 2024-11-28T01:41:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/72f455b93e354d3c8e4735ff895bbf20 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/72f455b93e354d3c8e4735ff895bbf20 2024-11-28T01:41:05,031 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/72f455b93e354d3c8e4735ff895bbf20, entries=150, sequenceid=459, filesize=12.0 K 2024-11-28T01:41:05,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/1ca4c6a5b4994b6c87ae0657fb26d518 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1ca4c6a5b4994b6c87ae0657fb26d518 2024-11-28T01:41:05,042 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1ca4c6a5b4994b6c87ae0657fb26d518, entries=150, sequenceid=459, filesize=12.0 K 2024-11-28T01:41:05,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/96e58dc3e9ff4909aafebe640e63e6a5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/96e58dc3e9ff4909aafebe640e63e6a5 2024-11-28T01:41:05,058 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/96e58dc3e9ff4909aafebe640e63e6a5, entries=150, sequenceid=459, filesize=12.0 K 2024-11-28T01:41:05,059 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.50 KB/456192, currentSize=26.84 KB/27480 for 707ae842769f4709bfcd7372850df285 in 524ms, sequenceid=459, compaction requested=true 2024-11-28T01:41:05,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:05,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-28T01:41:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-28T01:41:05,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T01:41:05,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6110 sec 2024-11-28T01:41:05,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6220 sec 2024-11-28T01:41:05,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:05,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:05,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:05,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:05,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/9d9b3a974ad94ef18641e162244c5a0d is 50, key is test_row_0/A:col10/1732758064669/Put/seqid=0 2024-11-28T01:41:05,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741885_1061 (size=14741) 2024-11-28T01:41:05,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/9d9b3a974ad94ef18641e162244c5a0d 2024-11-28T01:41:05,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d45a56d444c3469c99b95cf750acfafd is 50, key is test_row_0/B:col10/1732758064669/Put/seqid=0 2024-11-28T01:41:05,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741886_1062 (size=12301) 2024-11-28T01:41:05,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d45a56d444c3469c99b95cf750acfafd 2024-11-28T01:41:05,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7461f245e0cb4eb1af107141c4a53997 is 50, key is test_row_0/C:col10/1732758064669/Put/seqid=0 2024-11-28T01:41:05,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741887_1063 (size=12301) 2024-11-28T01:41:05,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7461f245e0cb4eb1af107141c4a53997 2024-11-28T01:41:05,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758065254, max timestamp=1732758065310Num uniques 859; 2024-11-28T01:41:05,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:05,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758065254, max timestamp=1732758065310Num uniques 859; 2024-11-28T01:41:05,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758065254, max timestamp=1732758065310Num uniques 859; 2024-11-28T01:41:05,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:05,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/9d9b3a974ad94ef18641e162244c5a0d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d9b3a974ad94ef18641e162244c5a0d 2024-11-28T01:41:05,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d9b3a974ad94ef18641e162244c5a0d, entries=200, sequenceid=469, filesize=14.4 K 2024-11-28T01:41:05,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d45a56d444c3469c99b95cf750acfafd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d45a56d444c3469c99b95cf750acfafd 2024-11-28T01:41:05,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d45a56d444c3469c99b95cf750acfafd, entries=150, sequenceid=469, filesize=12.0 K 2024-11-28T01:41:05,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7461f245e0cb4eb1af107141c4a53997 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7461f245e0cb4eb1af107141c4a53997 2024-11-28T01:41:05,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7461f245e0cb4eb1af107141c4a53997, entries=150, sequenceid=469, filesize=12.0 K 2024-11-28T01:41:05,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 707ae842769f4709bfcd7372850df285 in 163ms, sequenceid=469, compaction requested=true 2024-11-28T01:41:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:05,356 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:05,356 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:05,358 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:05,358 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:05,358 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,358 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:05,359 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:05,359 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/de2bb5c72f8d460da4387f020a5605cd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/3665f894bc1d48cd89ad94d8e6690fb6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/72f455b93e354d3c8e4735ff895bbf20, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d9b3a974ad94ef18641e162244c5a0d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=50.9 K 2024-11-28T01:41:05,359 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,359 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/91914a43f7c9432dac65590c786e0153, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/997e283f1dba4c0f81ffc07b570f63d7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1ca4c6a5b4994b6c87ae0657fb26d518, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d45a56d444c3469c99b95cf750acfafd] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=48.5 K 2024-11-28T01:41:05,359 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 91914a43f7c9432dac65590c786e0153, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732758062756 2024-11-28T01:41:05,360 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting de2bb5c72f8d460da4387f020a5605cd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732758062756 2024-11-28T01:41:05,360 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3665f894bc1d48cd89ad94d8e6690fb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732758062859 2024-11-28T01:41:05,361 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 997e283f1dba4c0f81ffc07b570f63d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732758062859 2024-11-28T01:41:05,361 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72f455b93e354d3c8e4735ff895bbf20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732758064011 2024-11-28T01:41:05,361 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ca4c6a5b4994b6c87ae0657fb26d518, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732758064011 2024-11-28T01:41:05,362 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9b3a974ad94ef18641e162244c5a0d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732758064669 2024-11-28T01:41:05,362 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d45a56d444c3469c99b95cf750acfafd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732758064669 2024-11-28T01:41:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:05,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T01:41:05,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758065338, max timestamp=1732758065364Num uniques 1068; 2024-11-28T01:41:05,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-28T01:41:05,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758065338, max timestamp=1732758065364Num uniques 1068; 2024-11-28T01:41:05,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-28T01:41:05,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758065338, max timestamp=1732758065364Num uniques 1068; 2024-11-28T01:41:05,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-28T01:41:05,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:05,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:05,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:05,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:05,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:05,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:05,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/dd1ffcd4e9e943c2bfd2c4a5b828b384 is 50, key is test_row_0/A:col10/1732758065353/Put/seqid=0 2024-11-28T01:41:05,395 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#53 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:05,395 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/d1fe55ec7c3a414f882c404d3ceebb81 is 50, key is test_row_0/A:col10/1732758064669/Put/seqid=0 2024-11-28T01:41:05,400 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#51 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:05,402 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/0700e1ab4a894df8870e752405055362 is 50, key is test_row_0/B:col10/1732758064669/Put/seqid=0 2024-11-28T01:41:05,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732758065396, max timestamp=1732758065420Num uniques 859; 2024-11-28T01:41:05,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-28T01:41:05,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732758065396, max timestamp=1732758065420Num uniques 859; 2024-11-28T01:41:05,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-28T01:41:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758065411, max timestamp=1732758065420Num uniques 859; 2024-11-28T01:41:05,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T01:41:05,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741888_1064 (size=12301) 2024-11-28T01:41:05,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/dd1ffcd4e9e943c2bfd2c4a5b828b384 2024-11-28T01:41:05,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7821ef34a2674a769d3b3ae65afee4ad is 50, key is test_row_0/B:col10/1732758065353/Put/seqid=0 2024-11-28T01:41:05,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741889_1065 (size=12881) 2024-11-28T01:41:05,475 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/d1fe55ec7c3a414f882c404d3ceebb81 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/d1fe55ec7c3a414f882c404d3ceebb81 2024-11-28T01:41:05,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741890_1066 (size=12881) 2024-11-28T01:41:05,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758125461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758125463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758065459, max timestamp=1732758065477Num uniques 1268; 2024-11-28T01:41:05,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=9978, suffix heap size=159840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=218 2024-11-28T01:41:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758065459, max timestamp=1732758065477Num uniques 1268; 2024-11-28T01:41:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=58068, new segment data size=7688, suffix heap size=159840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1268, new segment cells count=168 2024-11-28T01:41:05,488 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/0700e1ab4a894df8870e752405055362 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/0700e1ab4a894df8870e752405055362 2024-11-28T01:41:05,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758065459, max timestamp=1732758065483Num uniques 1009; 2024-11-28T01:41:05,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-28T01:41:05,496 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into d1fe55ec7c3a414f882c404d3ceebb81(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:05,496 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:05,497 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=12, startTime=1732758065356; duration=0sec 2024-11-28T01:41:05,497 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into 0700e1ab4a894df8870e752405055362(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:05,497 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:05,497 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:05,497 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=12, startTime=1732758065356; duration=0sec 2024-11-28T01:41:05,498 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:05,498 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:05,498 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:05,497 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:05,502 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:05,502 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:05,502 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,502 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/031278f9365e4b3ea610501083d3600c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/5585104e81344c548872da8d261565cf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/96e58dc3e9ff4909aafebe640e63e6a5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7461f245e0cb4eb1af107141c4a53997] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=48.5 K 2024-11-28T01:41:05,503 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 031278f9365e4b3ea610501083d3600c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732758062756 2024-11-28T01:41:05,504 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5585104e81344c548872da8d261565cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732758062859 2024-11-28T01:41:05,505 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e58dc3e9ff4909aafebe640e63e6a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732758064011 2024-11-28T01:41:05,506 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7461f245e0cb4eb1af107141c4a53997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732758064669 2024-11-28T01:41:05,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741891_1067 (size=12301) 2024-11-28T01:41:05,527 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:05,528 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a15fe028fb9446a9889023a14faebde4 is 50, key is test_row_0/C:col10/1732758064669/Put/seqid=0 2024-11-28T01:41:05,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741892_1068 (size=12881) 2024-11-28T01:41:05,544 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a15fe028fb9446a9889023a14faebde4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a15fe028fb9446a9889023a14faebde4 2024-11-28T01:41:05,554 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into a15fe028fb9446a9889023a14faebde4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:05,554 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:05,555 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=12, startTime=1732758065356; duration=0sec 2024-11-28T01:41:05,555 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:05,555 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:05,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732758065537, max timestamp=1732758065565Num uniques 1077; 2024-11-28T01:41:05,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732758065537, max timestamp=1732758065565Num uniques 1027; 2024-11-28T01:41:05,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-28T01:41:05,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-28T01:41:05,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732758065546, max timestamp=1732758065565Num uniques 1027; 2024-11-28T01:41:05,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-28T01:41:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T01:41:05,593 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:05,598 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-28T01:41:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T01:41:05,601 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:05,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:05,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:05,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1732758065596, max timestamp=1732758065617Num uniques 1136; 2024-11-28T01:41:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-11-28T01:41:05,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1732758065596, max timestamp=1732758065617Num uniques 1136; 2024-11-28T01:41:05,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-11-28T01:41:05,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1732758065596, max timestamp=1732758065617Num uniques 1036; 2024-11-28T01:41:05,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=17676, suffix heap size=135712,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=386 2024-11-28T01:41:05,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758125662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758125665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758125661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758125669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758125671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T01:41:05,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:05,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:05,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:05,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:05,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758125782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758125782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758125783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758125793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758125793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:05,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:05,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:05,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:05,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T01:41:05,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7821ef34a2674a769d3b3ae65afee4ad 2024-11-28T01:41:05,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/9561c729e1614c6e87663801f4983d84 is 50, key is test_row_0/C:col10/1732758065353/Put/seqid=0 2024-11-28T01:41:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741893_1069 (size=12301) 2024-11-28T01:41:05,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/9561c729e1614c6e87663801f4983d84 2024-11-28T01:41:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/dd1ffcd4e9e943c2bfd2c4a5b828b384 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/dd1ffcd4e9e943c2bfd2c4a5b828b384 2024-11-28T01:41:05,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/dd1ffcd4e9e943c2bfd2c4a5b828b384, entries=150, sequenceid=507, filesize=12.0 K 2024-11-28T01:41:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7821ef34a2674a769d3b3ae65afee4ad as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7821ef34a2674a769d3b3ae65afee4ad 2024-11-28T01:41:05,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7821ef34a2674a769d3b3ae65afee4ad, entries=150, sequenceid=507, filesize=12.0 K 2024-11-28T01:41:05,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/9561c729e1614c6e87663801f4983d84 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/9561c729e1614c6e87663801f4983d84 2024-11-28T01:41:05,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758125993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758125993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758125994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:05,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:05,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/9561c729e1614c6e87663801f4983d84, entries=150, sequenceid=507, filesize=12.0 K 2024-11-28T01:41:06,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=161.02 KB/164880 for 707ae842769f4709bfcd7372850df285 in 641ms, sequenceid=507, compaction requested=false 2024-11-28T01:41:06,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:06,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732758065633, max timestamp=1732758066006Num uniques 1245; 2024-11-28T01:41:06,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=8930, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-28T01:41:06,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732758065633, max timestamp=1732758066006Num uniques 1245; 2024-11-28T01:41:06,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=8930, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-28T01:41:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:06,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=73.80 KB heapSize=196.58 KB 2024-11-28T01:41:06,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732758065633, max timestamp=1732758066009Num uniques 1245; 2024-11-28T01:41:06,014 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=8930, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-28T01:41:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:06,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/4a99ae3a28d74e169f8a833c530a9088 is 50, key is test_row_0/A:col10/1732758066006/Put/seqid=0 2024-11-28T01:41:06,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741894_1070 (size=14741) 2024-11-28T01:41:06,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/4a99ae3a28d74e169f8a833c530a9088 2024-11-28T01:41:06,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/a953b083226a45b4bfd85b97e8ceec4f is 50, key is test_row_0/B:col10/1732758066006/Put/seqid=0 2024-11-28T01:41:06,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741895_1071 (size=12301) 2024-11-28T01:41:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758066073, max timestamp=1732758066089Num uniques 859; 2024-11-28T01:41:06,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:06,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758066073, max timestamp=1732758066089Num uniques 859; 2024-11-28T01:41:06,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:06,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758066073, max timestamp=1732758066089Num uniques 859; 2024-11-28T01:41:06,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:06,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7687, totalHeapSize=17712, min timestamp=1732758066125, max timestamp=1732758066147Num uniques 1118; 2024-11-28T01:41:06,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51197, new segment data size=7687, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-28T01:41:06,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7684, totalHeapSize=17712, min timestamp=1732758066125, max timestamp=1732758066147Num uniques 1118; 2024-11-28T01:41:06,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51194, new segment data size=7684, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-28T01:41:06,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7682, totalHeapSize=17712, min timestamp=1732758066125, max timestamp=1732758066147Num uniques 1118; 2024-11-28T01:41:06,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51192, new segment data size=7682, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-28T01:41:06,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732758066194, max timestamp=1732758066208Num uniques 1027; 2024-11-28T01:41:06,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732758066194, max timestamp=1732758066208Num uniques 1027; 2024-11-28T01:41:06,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732758066194, max timestamp=1732758066208Num uniques 1027; 2024-11-28T01:41:06,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-28T01:41:06,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-28T01:41:06,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-28T01:41:06,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:06,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:06,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T01:41:06,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758126269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732758066244, max timestamp=1732758066269Num uniques 1136; 2024-11-28T01:41:06,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732758066244, max timestamp=1732758066269Num uniques 1136; 2024-11-28T01:41:06,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-28T01:41:06,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-28T01:41:06,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732758066244, max timestamp=1732758066269Num uniques 1136; 2024-11-28T01:41:06,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-28T01:41:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8912, totalHeapSize=20520, min timestamp=1732758066320, max timestamp=1732758066344Num uniques 1195; 2024-11-28T01:41:06,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54712, new segment data size=8912, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-28T01:41:06,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1732758066320, max timestamp=1732758066338Num uniques 1195; 2024-11-28T01:41:06,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54720, new segment data size=8920, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-28T01:41:06,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8910, totalHeapSize=20520, min timestamp=1732758066320, max timestamp=1732758066344Num uniques 1195; 2024-11-28T01:41:06,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54710, new segment data size=8910, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-28T01:41:06,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:06,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=454, cellSize=20790, totalHeapSize=47456, min timestamp=1732758066384, max timestamp=1732758066400Num uniques 1054; 2024-11-28T01:41:06,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=20790, suffix heap size=137584,new segment heap size=47456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=454 2024-11-28T01:41:06,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758126396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=454, cellSize=20790, totalHeapSize=47456, min timestamp=1732758066384, max timestamp=1732758066400Num uniques 1054; 2024-11-28T01:41:06,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=20790, suffix heap size=137584,new segment heap size=47456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=454 2024-11-28T01:41:06,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758066389, max timestamp=1732758066400Num uniques 1054; 2024-11-28T01:41:06,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=6870, suffix heap size=137584,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=150 2024-11-28T01:41:06,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758126432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758126436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758126437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758126438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/a953b083226a45b4bfd85b97e8ceec4f 2024-11-28T01:41:06,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/6109f550e70940a39918b81d9e0dd8ce is 50, key is test_row_0/C:col10/1732758066006/Put/seqid=0 2024-11-28T01:41:06,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741896_1072 (size=12301) 2024-11-28T01:41:06,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758126513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:06,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758126542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758126542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758126552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758126553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:06,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:06,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758126722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T01:41:06,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758126753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758126762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758126762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758126762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:06,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:06,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/6109f550e70940a39918b81d9e0dd8ce 2024-11-28T01:41:06,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/4a99ae3a28d74e169f8a833c530a9088 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4a99ae3a28d74e169f8a833c530a9088 2024-11-28T01:41:06,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4a99ae3a28d74e169f8a833c530a9088, entries=200, sequenceid=600, filesize=14.4 K 2024-11-28T01:41:06,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/a953b083226a45b4bfd85b97e8ceec4f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/a953b083226a45b4bfd85b97e8ceec4f 2024-11-28T01:41:06,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/a953b083226a45b4bfd85b97e8ceec4f, entries=150, sequenceid=600, filesize=12.0 K 2024-11-28T01:41:06,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/6109f550e70940a39918b81d9e0dd8ce as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6109f550e70940a39918b81d9e0dd8ce 2024-11-28T01:41:06,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6109f550e70940a39918b81d9e0dd8ce, entries=150, sequenceid=600, filesize=12.0 K 2024-11-28T01:41:06,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~82.73 KB/84720, currentSize=160.84 KB/164700 for 707ae842769f4709bfcd7372850df285 in 934ms, sequenceid=600, compaction requested=true 2024-11-28T01:41:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:06,949 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:06,951 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:06,951 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:06,951 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,951 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/0700e1ab4a894df8870e752405055362, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7821ef34a2674a769d3b3ae65afee4ad, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/a953b083226a45b4bfd85b97e8ceec4f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=36.6 K 2024-11-28T01:41:06,951 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:06,951 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0700e1ab4a894df8870e752405055362, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732758064669 2024-11-28T01:41:06,952 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7821ef34a2674a769d3b3ae65afee4ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732758065338 2024-11-28T01:41:06,953 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a953b083226a45b4bfd85b97e8ceec4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732758065633 2024-11-28T01:41:06,953 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:06,954 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:06,954 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,954 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/d1fe55ec7c3a414f882c404d3ceebb81, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/dd1ffcd4e9e943c2bfd2c4a5b828b384, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4a99ae3a28d74e169f8a833c530a9088] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=39.0 K 2024-11-28T01:41:06,955 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1fe55ec7c3a414f882c404d3ceebb81, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732758064669 2024-11-28T01:41:06,955 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1ffcd4e9e943c2bfd2c4a5b828b384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732758065338 2024-11-28T01:41:06,956 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a99ae3a28d74e169f8a833c530a9088, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732758065633 2024-11-28T01:41:06,964 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:06,965 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/371e0d67a63b43ee84cf54c3a93405b1 is 50, key is test_row_0/B:col10/1732758066006/Put/seqid=0 2024-11-28T01:41:06,971 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:06,971 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8e44ccf4cd284240bcfa52c6fb5356b9 is 50, key is test_row_0/A:col10/1732758066006/Put/seqid=0 2024-11-28T01:41:06,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741897_1073 (size=12983) 2024-11-28T01:41:06,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741898_1074 (size=12983) 2024-11-28T01:41:06,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T01:41:06,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:06,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=160.84 KB heapSize=445.98 KB 2024-11-28T01:41:06,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ed970bc1238c4894bd69ad3797995179 is 50, key is test_row_0/A:col10/1732758066433/Put/seqid=0 2024-11-28T01:41:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741899_1075 (size=12301) 2024-11-28T01:41:07,014 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=723 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ed970bc1238c4894bd69ad3797995179 2024-11-28T01:41:07,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/3edc8593bd7e43a39842f2bdc42bd93d is 50, key is test_row_0/B:col10/1732758066433/Put/seqid=0 2024-11-28T01:41:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:07,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:07,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741900_1076 (size=12301) 2024-11-28T01:41:07,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758127054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758127063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758127073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758127073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758127073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758127166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758127373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,386 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/371e0d67a63b43ee84cf54c3a93405b1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/371e0d67a63b43ee84cf54c3a93405b1 2024-11-28T01:41:07,386 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8e44ccf4cd284240bcfa52c6fb5356b9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8e44ccf4cd284240bcfa52c6fb5356b9 2024-11-28T01:41:07,394 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 8e44ccf4cd284240bcfa52c6fb5356b9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:07,395 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:07,395 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758066949; duration=0sec 2024-11-28T01:41:07,395 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:07,395 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:07,395 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:07,395 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into 371e0d67a63b43ee84cf54c3a93405b1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:07,395 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:07,396 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758066949; duration=0sec 2024-11-28T01:41:07,397 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:07,397 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:07,397 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:07,397 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:07,397 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:07,397 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a15fe028fb9446a9889023a14faebde4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/9561c729e1614c6e87663801f4983d84, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6109f550e70940a39918b81d9e0dd8ce] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=36.6 K 2024-11-28T01:41:07,398 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a15fe028fb9446a9889023a14faebde4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732758064669 2024-11-28T01:41:07,399 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9561c729e1614c6e87663801f4983d84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732758065338 2024-11-28T01:41:07,399 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6109f550e70940a39918b81d9e0dd8ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732758065633 2024-11-28T01:41:07,410 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:07,412 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/e69b650dfa7b4a819aca050070ce9ae2 is 50, key is test_row_0/C:col10/1732758066006/Put/seqid=0 2024-11-28T01:41:07,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741901_1077 (size=12983) 2024-11-28T01:41:07,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.55 KB at sequenceid=723 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/3edc8593bd7e43a39842f2bdc42bd93d 2024-11-28T01:41:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a47ec7c838d449c3933f7259de8980f2 is 50, key is test_row_0/C:col10/1732758066433/Put/seqid=0 2024-11-28T01:41:07,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741902_1078 (size=12301) 2024-11-28T01:41:07,467 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=723 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a47ec7c838d449c3933f7259de8980f2 2024-11-28T01:41:07,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/ed970bc1238c4894bd69ad3797995179 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ed970bc1238c4894bd69ad3797995179 2024-11-28T01:41:07,496 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ed970bc1238c4894bd69ad3797995179, entries=150, sequenceid=723, filesize=12.0 K 2024-11-28T01:41:07,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/3edc8593bd7e43a39842f2bdc42bd93d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3edc8593bd7e43a39842f2bdc42bd93d 2024-11-28T01:41:07,513 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3edc8593bd7e43a39842f2bdc42bd93d, entries=150, sequenceid=723, filesize=12.0 K 2024-11-28T01:41:07,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/a47ec7c838d449c3933f7259de8980f2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a47ec7c838d449c3933f7259de8980f2 2024-11-28T01:41:07,534 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a47ec7c838d449c3933f7259de8980f2, entries=150, sequenceid=723, filesize=12.0 K 2024-11-28T01:41:07,536 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~160.84 KB/164700, heapSize ~445.94 KB/456640, currentSize=26.84 KB/27480 for 707ae842769f4709bfcd7372850df285 in 544ms, sequenceid=723, compaction requested=false 2024-11-28T01:41:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:07,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-28T01:41:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-28T01:41:07,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-28T01:41:07,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-11-28T01:41:07,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9480 sec 2024-11-28T01:41:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:07,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:07,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:07,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:07,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:07,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:07,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:07,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:07,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/90c945b11227414ab4c6f622485121cf is 50, key is test_row_1/A:col10/1732758067579/Put/seqid=0 2024-11-28T01:41:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741903_1079 (size=9857) 2024-11-28T01:41:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758067606, max timestamp=1732758067649Num uniques 859; 2024-11-28T01:41:07,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:07,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758067606, max timestamp=1732758067649Num uniques 859; 2024-11-28T01:41:07,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758127647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758127648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758067606, max timestamp=1732758067649Num uniques 859; 2024-11-28T01:41:07,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T01:41:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758127715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758127724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758127726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T01:41:07,742 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:07,743 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-28T01:41:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T01:41:07,745 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:07,747 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:07,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758127763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758127762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,835 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/e69b650dfa7b4a819aca050070ce9ae2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e69b650dfa7b4a819aca050070ce9ae2 2024-11-28T01:41:07,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758127832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758127833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758127833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,844 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into e69b650dfa7b4a819aca050070ce9ae2(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:07,844 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:07,844 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758066949; duration=0sec 2024-11-28T01:41:07,845 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:07,845 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T01:41:07,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T01:41:07,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:07,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:07,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:07,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:07,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:07,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758127972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758127972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/90c945b11227414ab4c6f622485121cf 2024-11-28T01:41:08,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/68792f3d2756425baff7a7e8a94e8d2e is 50, key is test_row_1/B:col10/1732758067579/Put/seqid=0 2024-11-28T01:41:08,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741904_1080 (size=9857) 2024-11-28T01:41:08,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/68792f3d2756425baff7a7e8a94e8d2e 2024-11-28T01:41:08,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758128042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/f55b180236da42c98039c8a7c9496445 is 50, key is test_row_1/C:col10/1732758067579/Put/seqid=0 2024-11-28T01:41:08,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:08,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758128053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758128053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:08,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741905_1081 (size=9857) 2024-11-28T01:41:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T01:41:08,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/f55b180236da42c98039c8a7c9496445 2024-11-28T01:41:08,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/90c945b11227414ab4c6f622485121cf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/90c945b11227414ab4c6f622485121cf 2024-11-28T01:41:08,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/90c945b11227414ab4c6f622485121cf, entries=100, sequenceid=736, filesize=9.6 K 2024-11-28T01:41:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/68792f3d2756425baff7a7e8a94e8d2e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/68792f3d2756425baff7a7e8a94e8d2e 2024-11-28T01:41:08,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/68792f3d2756425baff7a7e8a94e8d2e, entries=100, sequenceid=736, filesize=9.6 K 2024-11-28T01:41:08,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/f55b180236da42c98039c8a7c9496445 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f55b180236da42c98039c8a7c9496445 2024-11-28T01:41:08,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f55b180236da42c98039c8a7c9496445, entries=100, sequenceid=736, filesize=9.6 K 2024-11-28T01:41:08,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 707ae842769f4709bfcd7372850df285 in 523ms, sequenceid=736, compaction requested=true 2024-11-28T01:41:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:08,108 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:08,108 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:08,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:08,113 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:08,113 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:08,113 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:08,113 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8e44ccf4cd284240bcfa52c6fb5356b9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ed970bc1238c4894bd69ad3797995179, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/90c945b11227414ab4c6f622485121cf] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=34.3 K 2024-11-28T01:41:08,114 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:08,114 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e44ccf4cd284240bcfa52c6fb5356b9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732758065633 2024-11-28T01:41:08,114 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:08,115 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:08,115 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed970bc1238c4894bd69ad3797995179, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=723, earliestPutTs=1732758066429 2024-11-28T01:41:08,115 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/371e0d67a63b43ee84cf54c3a93405b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3edc8593bd7e43a39842f2bdc42bd93d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/68792f3d2756425baff7a7e8a94e8d2e] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=34.3 K 2024-11-28T01:41:08,115 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c945b11227414ab4c6f622485121cf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732758067579 2024-11-28T01:41:08,116 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 371e0d67a63b43ee84cf54c3a93405b1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732758065633 2024-11-28T01:41:08,117 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3edc8593bd7e43a39842f2bdc42bd93d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=723, earliestPutTs=1732758066429 2024-11-28T01:41:08,117 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 68792f3d2756425baff7a7e8a94e8d2e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732758067579 2024-11-28T01:41:08,142 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:08,143 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/2b751de78cac49598d3a02845f13602c is 50, key is test_row_0/A:col10/1732758066433/Put/seqid=0 2024-11-28T01:41:08,152 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:08,153 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f4537fceddef43faaba0e5f6a729641b is 50, key is test_row_0/B:col10/1732758066433/Put/seqid=0 2024-11-28T01:41:08,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741906_1082 (size=13085) 2024-11-28T01:41:08,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741907_1083 (size=13085) 2024-11-28T01:41:08,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T01:41:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:08,210 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-28T01:41:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:08,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/c7d85858da384086b94d16f8a7fdc67a is 50, key is test_row_0/A:col10/1732758067708/Put/seqid=0 2024-11-28T01:41:08,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741908_1084 (size=12301) 2024-11-28T01:41:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:08,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758128328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758128335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758128352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758128363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758128363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T01:41:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758128443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758128452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,577 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f4537fceddef43faaba0e5f6a729641b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f4537fceddef43faaba0e5f6a729641b 2024-11-28T01:41:08,578 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/2b751de78cac49598d3a02845f13602c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/2b751de78cac49598d3a02845f13602c 2024-11-28T01:41:08,592 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 2b751de78cac49598d3a02845f13602c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:08,592 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:08,592 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758068108; duration=0sec 2024-11-28T01:41:08,592 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:08,592 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:08,592 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:08,593 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into f4537fceddef43faaba0e5f6a729641b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:08,593 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:08,593 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758068108; duration=0sec 2024-11-28T01:41:08,595 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:08,596 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:08,597 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:08,597 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:08,597 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:08,597 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e69b650dfa7b4a819aca050070ce9ae2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a47ec7c838d449c3933f7259de8980f2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f55b180236da42c98039c8a7c9496445] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=34.3 K 2024-11-28T01:41:08,598 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e69b650dfa7b4a819aca050070ce9ae2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732758065633 2024-11-28T01:41:08,598 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a47ec7c838d449c3933f7259de8980f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=723, earliestPutTs=1732758066429 2024-11-28T01:41:08,599 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55b180236da42c98039c8a7c9496445, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732758067579 2024-11-28T01:41:08,625 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:08,626 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/ae54f3dc30d54867863a690e4af37eae is 50, key is test_row_0/C:col10/1732758066433/Put/seqid=0 2024-11-28T01:41:08,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741909_1085 (size=13085) 2024-11-28T01:41:08,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/c7d85858da384086b94d16f8a7fdc67a 2024-11-28T01:41:08,654 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/ae54f3dc30d54867863a690e4af37eae as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/ae54f3dc30d54867863a690e4af37eae 2024-11-28T01:41:08,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758128653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758128662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/9b127700ea774ef09d88a6273d8697b5 is 50, key is test_row_0/B:col10/1732758067708/Put/seqid=0 2024-11-28T01:41:08,670 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into ae54f3dc30d54867863a690e4af37eae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:08,670 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:08,670 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758068108; duration=0sec 2024-11-28T01:41:08,670 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:08,670 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:08,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741910_1086 (size=12301) 2024-11-28T01:41:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758128872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758128872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758128872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T01:41:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758128972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758128972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,083 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/9b127700ea774ef09d88a6273d8697b5 2024-11-28T01:41:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/bfff018a884e48cd9fa6a7e2d917d6a1 is 50, key is test_row_0/C:col10/1732758067708/Put/seqid=0 2024-11-28T01:41:09,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741911_1087 (size=12301) 2024-11-28T01:41:09,110 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/bfff018a884e48cd9fa6a7e2d917d6a1 2024-11-28T01:41:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/c7d85858da384086b94d16f8a7fdc67a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c7d85858da384086b94d16f8a7fdc67a 2024-11-28T01:41:09,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c7d85858da384086b94d16f8a7fdc67a, entries=150, sequenceid=773, filesize=12.0 K 2024-11-28T01:41:09,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/9b127700ea774ef09d88a6273d8697b5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/9b127700ea774ef09d88a6273d8697b5 2024-11-28T01:41:09,139 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/9b127700ea774ef09d88a6273d8697b5, entries=150, sequenceid=773, filesize=12.0 K 2024-11-28T01:41:09,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/bfff018a884e48cd9fa6a7e2d917d6a1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bfff018a884e48cd9fa6a7e2d917d6a1 2024-11-28T01:41:09,146 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bfff018a884e48cd9fa6a7e2d917d6a1, entries=150, sequenceid=773, filesize=12.0 K 2024-11-28T01:41:09,147 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=53.67 KB/54960 for 707ae842769f4709bfcd7372850df285 in 937ms, sequenceid=773, compaction requested=false 2024-11-28T01:41:09,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-28T01:41:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-28T01:41:09,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T01:41:09,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-28T01:41:09,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.4110 sec 2024-11-28T01:41:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:09,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:41:09,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:09,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:09,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:09,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:09,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:09,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:09,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/840cdc6e12e44853ad49ad69d417c1ee is 50, key is test_row_0/A:col10/1732758068293/Put/seqid=0 2024-11-28T01:41:09,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741912_1088 (size=14741) 2024-11-28T01:41:09,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758129536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758129539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758129653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758129653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758129863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758129863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T01:41:09,894 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:09,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-28T01:41:09,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758129893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T01:41:09,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758129893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:09,897 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758129893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,898 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,899 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:09,899 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:09,905 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:09,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:09,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/840cdc6e12e44853ad49ad69d417c1ee 2024-11-28T01:41:09,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d67156137d6247198d68a9b82d6d893d is 50, key is test_row_0/B:col10/1732758068293/Put/seqid=0 2024-11-28T01:41:09,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741913_1089 (size=12301) 2024-11-28T01:41:09,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d67156137d6247198d68a9b82d6d893d 2024-11-28T01:41:09,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/c0e3d9db1e2d4a3cb808e8496f22410c is 50, key is test_row_0/C:col10/1732758068293/Put/seqid=0 2024-11-28T01:41:10,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T01:41:10,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741914_1090 (size=12301) 2024-11-28T01:41:10,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=788 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/c0e3d9db1e2d4a3cb808e8496f22410c 2024-11-28T01:41:10,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/840cdc6e12e44853ad49ad69d417c1ee as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/840cdc6e12e44853ad49ad69d417c1ee 2024-11-28T01:41:10,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/840cdc6e12e44853ad49ad69d417c1ee, entries=200, sequenceid=788, filesize=14.4 K 2024-11-28T01:41:10,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d67156137d6247198d68a9b82d6d893d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d67156137d6247198d68a9b82d6d893d 2024-11-28T01:41:10,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d67156137d6247198d68a9b82d6d893d, entries=150, sequenceid=788, filesize=12.0 K 2024-11-28T01:41:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/c0e3d9db1e2d4a3cb808e8496f22410c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/c0e3d9db1e2d4a3cb808e8496f22410c 2024-11-28T01:41:10,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/c0e3d9db1e2d4a3cb808e8496f22410c, entries=150, sequenceid=788, filesize=12.0 K 2024-11-28T01:41:10,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 707ae842769f4709bfcd7372850df285 in 573ms, sequenceid=788, compaction requested=true 2024-11-28T01:41:10,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:10,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:10,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:10,059 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:10,060 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:10,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T01:41:10,062 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:10,062 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:10,062 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:10,062 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/2b751de78cac49598d3a02845f13602c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c7d85858da384086b94d16f8a7fdc67a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/840cdc6e12e44853ad49ad69d417c1ee] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=39.2 K 2024-11-28T01:41:10,063 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:10,063 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:10,063 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:10,063 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f4537fceddef43faaba0e5f6a729641b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/9b127700ea774ef09d88a6273d8697b5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d67156137d6247198d68a9b82d6d893d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=36.8 K 2024-11-28T01:41:10,064 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b751de78cac49598d3a02845f13602c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732758066433 2024-11-28T01:41:10,064 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4537fceddef43faaba0e5f6a729641b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732758066433 2024-11-28T01:41:10,064 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d85858da384086b94d16f8a7fdc67a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1732758067708 2024-11-28T01:41:10,065 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b127700ea774ef09d88a6273d8697b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1732758067708 2024-11-28T01:41:10,065 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 840cdc6e12e44853ad49ad69d417c1ee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732758068293 2024-11-28T01:41:10,066 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d67156137d6247198d68a9b82d6d893d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732758068293 2024-11-28T01:41:10,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:10,068 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:41:10,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:10,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8cbc6b2b99d949819eac5acf50777f2f is 50, key is test_row_0/A:col10/1732758069528/Put/seqid=0 2024-11-28T01:41:10,091 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:10,092 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/2836ac874c434a428b69bb1ae8d1a772 is 50, key is test_row_0/B:col10/1732758068293/Put/seqid=0 2024-11-28T01:41:10,098 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:10,099 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/43e73f2d1a2d44afb84d7742d9a47a98 is 50, key is test_row_0/A:col10/1732758068293/Put/seqid=0 2024-11-28T01:41:10,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741915_1091 (size=12301) 2024-11-28T01:41:10,124 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=808 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8cbc6b2b99d949819eac5acf50777f2f 2024-11-28T01:41:10,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741916_1092 (size=13187) 2024-11-28T01:41:10,153 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/2836ac874c434a428b69bb1ae8d1a772 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/2836ac874c434a428b69bb1ae8d1a772 2024-11-28T01:41:10,165 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into 2836ac874c434a428b69bb1ae8d1a772(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:10,166 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:10,166 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758070060; duration=0sec 2024-11-28T01:41:10,166 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:10,166 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:10,166 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:10,167 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:10,167 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:10,167 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:10,168 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/ae54f3dc30d54867863a690e4af37eae, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bfff018a884e48cd9fa6a7e2d917d6a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/c0e3d9db1e2d4a3cb808e8496f22410c] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=36.8 K 2024-11-28T01:41:10,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae54f3dc30d54867863a690e4af37eae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=736, earliestPutTs=1732758066433 2024-11-28T01:41:10,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d8ff34194e324d688dfdbec6f6d42c5a is 50, key is test_row_0/B:col10/1732758069528/Put/seqid=0 2024-11-28T01:41:10,172 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfff018a884e48cd9fa6a7e2d917d6a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1732758067708 2024-11-28T01:41:10,173 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e3d9db1e2d4a3cb808e8496f22410c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732758068293 2024-11-28T01:41:10,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741917_1093 (size=13187) 2024-11-28T01:41:10,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:10,187 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/43e73f2d1a2d44afb84d7742d9a47a98 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/43e73f2d1a2d44afb84d7742d9a47a98 2024-11-28T01:41:10,195 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 43e73f2d1a2d44afb84d7742d9a47a98(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:10,195 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:10,196 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758070059; duration=0sec 2024-11-28T01:41:10,196 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:10,196 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:10,205 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:10,205 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/f24ba7cb18e14da2b81903f4048afecb is 50, key is test_row_0/C:col10/1732758068293/Put/seqid=0 2024-11-28T01:41:10,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741918_1094 (size=12301) 2024-11-28T01:41:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T01:41:10,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758130225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758130225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:10,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741919_1095 (size=13187) 2024-11-28T01:41:10,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758130356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758130357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T01:41:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758130563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758130569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:10,608 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=808 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d8ff34194e324d688dfdbec6f6d42c5a 2024-11-28T01:41:10,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/6c0dc0223ea24e34bd67720fbf92a95e is 50, key is test_row_0/C:col10/1732758069528/Put/seqid=0 2024-11-28T01:41:10,643 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/f24ba7cb18e14da2b81903f4048afecb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f24ba7cb18e14da2b81903f4048afecb 2024-11-28T01:41:10,653 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into f24ba7cb18e14da2b81903f4048afecb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:10,654 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:10,654 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758070060; duration=0sec 2024-11-28T01:41:10,654 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:10,654 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:10,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741920_1096 (size=12301) 2024-11-28T01:41:10,663 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=808 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/6c0dc0223ea24e34bd67720fbf92a95e 2024-11-28T01:41:10,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8cbc6b2b99d949819eac5acf50777f2f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8cbc6b2b99d949819eac5acf50777f2f 2024-11-28T01:41:10,681 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8cbc6b2b99d949819eac5acf50777f2f, entries=150, sequenceid=808, filesize=12.0 K 2024-11-28T01:41:10,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/d8ff34194e324d688dfdbec6f6d42c5a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d8ff34194e324d688dfdbec6f6d42c5a 2024-11-28T01:41:10,695 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d8ff34194e324d688dfdbec6f6d42c5a, entries=150, sequenceid=808, filesize=12.0 K 2024-11-28T01:41:10,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/6c0dc0223ea24e34bd67720fbf92a95e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6c0dc0223ea24e34bd67720fbf92a95e 2024-11-28T01:41:10,705 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6c0dc0223ea24e34bd67720fbf92a95e, entries=150, sequenceid=808, filesize=12.0 K 2024-11-28T01:41:10,706 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 707ae842769f4709bfcd7372850df285 in 638ms, sequenceid=808, compaction requested=false 2024-11-28T01:41:10,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:10,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:10,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-28T01:41:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-28T01:41:10,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T01:41:10,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 805 msec 2024-11-28T01:41:10,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 820 msec 2024-11-28T01:41:10,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:41:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:10,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:10,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:10,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:10,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:10,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:10,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:10,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/495a8bdc154a49e49b6d12b0348a3dde is 50, key is test_row_0/A:col10/1732758070885/Put/seqid=0 2024-11-28T01:41:10,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758130948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:10,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741921_1097 (size=14737) 2024-11-28T01:41:10,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758130956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T01:41:11,034 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:11,034 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-28T01:41:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T01:41:11,037 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:11,038 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:11,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:11,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758131068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758131073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T01:41:11,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T01:41:11,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:11,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758131282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758131283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T01:41:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T01:41:11,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=825 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/495a8bdc154a49e49b6d12b0348a3dde 2024-11-28T01:41:11,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4fa8b86f80ed4c99930893b787cbe3d4 is 50, key is test_row_0/B:col10/1732758070885/Put/seqid=0 2024-11-28T01:41:11,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741922_1098 (size=9857) 2024-11-28T01:41:11,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T01:41:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758131592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758131603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T01:41:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:11,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T01:41:11,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=825 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4fa8b86f80ed4c99930893b787cbe3d4 2024-11-28T01:41:11,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T01:41:11,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:11,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:11,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/83951d4a077e464f81b264db3f65eb43 is 50, key is test_row_0/C:col10/1732758070885/Put/seqid=0 2024-11-28T01:41:11,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741923_1099 (size=9857) 2024-11-28T01:41:11,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=825 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/83951d4a077e464f81b264db3f65eb43 2024-11-28T01:41:11,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/495a8bdc154a49e49b6d12b0348a3dde as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/495a8bdc154a49e49b6d12b0348a3dde 2024-11-28T01:41:11,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/495a8bdc154a49e49b6d12b0348a3dde, entries=200, sequenceid=825, filesize=14.4 K 2024-11-28T01:41:11,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/4fa8b86f80ed4c99930893b787cbe3d4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4fa8b86f80ed4c99930893b787cbe3d4 2024-11-28T01:41:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41344 deadline: 1732758131905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,914 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4fa8b86f80ed4c99930893b787cbe3d4, entries=100, sequenceid=825, filesize=9.6 K 2024-11-28T01:41:11,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41366 deadline: 1732758131913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41358 deadline: 1732758131913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:11,940 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,942 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 707ae842769f4709bfcd7372850df285 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:11,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:11,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:11,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/83951d4a077e464f81b264db3f65eb43 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/83951d4a077e464f81b264db3f65eb43 2024-11-28T01:41:11,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/83951d4a077e464f81b264db3f65eb43, entries=100, sequenceid=825, filesize=9.6 K 2024-11-28T01:41:11,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 707ae842769f4709bfcd7372850df285 in 1069ms, sequenceid=825, compaction requested=true 2024-11-28T01:41:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:11,957 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:11,957 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:11,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:11,958 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:11,958 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:11,958 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:11,958 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:11,958 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,959 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,959 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/43e73f2d1a2d44afb84d7742d9a47a98, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8cbc6b2b99d949819eac5acf50777f2f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/495a8bdc154a49e49b6d12b0348a3dde] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=39.3 K 2024-11-28T01:41:11,959 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/2836ac874c434a428b69bb1ae8d1a772, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d8ff34194e324d688dfdbec6f6d42c5a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4fa8b86f80ed4c99930893b787cbe3d4] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=34.5 K 2024-11-28T01:41:11,959 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e73f2d1a2d44afb84d7742d9a47a98, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732758068293 2024-11-28T01:41:11,959 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2836ac874c434a428b69bb1ae8d1a772, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732758068293 2024-11-28T01:41:11,960 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cbc6b2b99d949819eac5acf50777f2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=808, earliestPutTs=1732758069528 2024-11-28T01:41:11,960 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ff34194e324d688dfdbec6f6d42c5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=808, earliestPutTs=1732758069528 2024-11-28T01:41:11,960 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 495a8bdc154a49e49b6d12b0348a3dde, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1732758070222 2024-11-28T01:41:11,960 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa8b86f80ed4c99930893b787cbe3d4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1732758070885 2024-11-28T01:41:11,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T01:41:11,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:11,977 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T01:41:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:11,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:11,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:11,982 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:11,983 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f3d77f604eb048faba82fb04ce1b525f is 50, key is test_row_0/B:col10/1732758070885/Put/seqid=0 2024-11-28T01:41:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1383eab0fa8b4fd8b7287778714bb782 is 50, key is test_row_0/A:col10/1732758070953/Put/seqid=0 2024-11-28T01:41:11,994 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:11,995 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1a4dcc1724524b518b661b042db5b670 is 50, key is test_row_0/A:col10/1732758070885/Put/seqid=0 2024-11-28T01:41:12,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741924_1100 (size=13289) 2024-11-28T01:41:12,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741925_1101 (size=12301) 2024-11-28T01:41:12,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:12,116 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=843 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1383eab0fa8b4fd8b7287778714bb782 2024-11-28T01:41:12,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741926_1102 (size=13289) 2024-11-28T01:41:12,127 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1a4dcc1724524b518b661b042db5b670 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a4dcc1724524b518b661b042db5b670 2024-11-28T01:41:12,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/eaa5e1224b1d4753bb767a8b345d333d is 50, key is test_row_0/B:col10/1732758070953/Put/seqid=0 2024-11-28T01:41:12,139 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 1a4dcc1724524b518b661b042db5b670(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:12,139 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:12,139 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758071957; duration=0sec 2024-11-28T01:41:12,140 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:12,140 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:12,140 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:12,142 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:12,142 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:12,142 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:12,142 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f24ba7cb18e14da2b81903f4048afecb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6c0dc0223ea24e34bd67720fbf92a95e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/83951d4a077e464f81b264db3f65eb43] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=34.5 K 2024-11-28T01:41:12,143 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f24ba7cb18e14da2b81903f4048afecb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=788, earliestPutTs=1732758068293 2024-11-28T01:41:12,143 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c0dc0223ea24e34bd67720fbf92a95e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=808, earliestPutTs=1732758069528 2024-11-28T01:41:12,144 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83951d4a077e464f81b264db3f65eb43, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1732758070885 2024-11-28T01:41:12,173 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#91 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:12,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741927_1103 (size=12301) 2024-11-28T01:41:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T01:41:12,174 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/940a4ecb7817413aabf787d6a2ea5071 is 50, key is test_row_0/C:col10/1732758070885/Put/seqid=0 2024-11-28T01:41:12,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758132176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758132180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741928_1104 (size=13289) 2024-11-28T01:41:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758132293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758132293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,484 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/f3d77f604eb048faba82fb04ce1b525f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f3d77f604eb048faba82fb04ce1b525f 2024-11-28T01:41:12,495 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into f3d77f604eb048faba82fb04ce1b525f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:12,495 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:12,496 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758071957; duration=0sec 2024-11-28T01:41:12,496 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:12,496 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:12,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758132503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758132502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,571 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=843 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/eaa5e1224b1d4753bb767a8b345d333d 2024-11-28T01:41:12,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0e197c4de24045ee95d30d4c31f02cc9 is 50, key is test_row_0/C:col10/1732758070953/Put/seqid=0 2024-11-28T01:41:12,616 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/940a4ecb7817413aabf787d6a2ea5071 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/940a4ecb7817413aabf787d6a2ea5071 2024-11-28T01:41:12,623 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into 940a4ecb7817413aabf787d6a2ea5071(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:12,623 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:12,624 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758071957; duration=0sec 2024-11-28T01:41:12,624 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:12,624 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:12,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741929_1105 (size=12301) 2024-11-28T01:41:12,627 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=843 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0e197c4de24045ee95d30d4c31f02cc9 2024-11-28T01:41:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/1383eab0fa8b4fd8b7287778714bb782 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1383eab0fa8b4fd8b7287778714bb782 2024-11-28T01:41:12,643 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1383eab0fa8b4fd8b7287778714bb782, entries=150, sequenceid=843, filesize=12.0 K 2024-11-28T01:41:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/eaa5e1224b1d4753bb767a8b345d333d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/eaa5e1224b1d4753bb767a8b345d333d 2024-11-28T01:41:12,652 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/eaa5e1224b1d4753bb767a8b345d333d, entries=150, sequenceid=843, filesize=12.0 K 2024-11-28T01:41:12,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0e197c4de24045ee95d30d4c31f02cc9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0e197c4de24045ee95d30d4c31f02cc9 2024-11-28T01:41:12,666 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0e197c4de24045ee95d30d4c31f02cc9, entries=150, sequenceid=843, filesize=12.0 K 2024-11-28T01:41:12,667 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 707ae842769f4709bfcd7372850df285 in 690ms, sequenceid=843, compaction requested=false 2024-11-28T01:41:12,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-28T01:41:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-28T01:41:12,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-28T01:41:12,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-11-28T01:41:12,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6390 sec 2024-11-28T01:41:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:12,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T01:41:12,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:12,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:12,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:12,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:12,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:12,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:12,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/4111d5ff43f84c128aa0522daa394ad2 is 50, key is test_row_0/A:col10/1732758072814/Put/seqid=0 2024-11-28T01:41:12,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741930_1106 (size=14741) 2024-11-28T01:41:12,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=862 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/4111d5ff43f84c128aa0522daa394ad2 2024-11-28T01:41:12,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7873dedf4b544e6bbf54e03f8b46272a is 50, key is test_row_0/B:col10/1732758072814/Put/seqid=0 2024-11-28T01:41:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758132857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741931_1107 (size=12301) 2024-11-28T01:41:12,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=862 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7873dedf4b544e6bbf54e03f8b46272a 2024-11-28T01:41:12,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758132864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/d0a98de8eb8e4eb4b68f19aaf5135774 is 50, key is test_row_0/C:col10/1732758072814/Put/seqid=0 2024-11-28T01:41:12,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741932_1108 (size=12301) 2024-11-28T01:41:12,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=862 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/d0a98de8eb8e4eb4b68f19aaf5135774 2024-11-28T01:41:12,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/4111d5ff43f84c128aa0522daa394ad2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4111d5ff43f84c128aa0522daa394ad2 2024-11-28T01:41:12,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4111d5ff43f84c128aa0522daa394ad2, entries=200, sequenceid=862, filesize=14.4 K 2024-11-28T01:41:12,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/7873dedf4b544e6bbf54e03f8b46272a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7873dedf4b544e6bbf54e03f8b46272a 2024-11-28T01:41:12,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7873dedf4b544e6bbf54e03f8b46272a, entries=150, sequenceid=862, filesize=12.0 K 2024-11-28T01:41:12,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/d0a98de8eb8e4eb4b68f19aaf5135774 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/d0a98de8eb8e4eb4b68f19aaf5135774 2024-11-28T01:41:12,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758132972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758132983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:12,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/d0a98de8eb8e4eb4b68f19aaf5135774, entries=150, sequenceid=862, filesize=12.0 K 2024-11-28T01:41:12,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 707ae842769f4709bfcd7372850df285 in 175ms, sequenceid=862, compaction requested=true 2024-11-28T01:41:12,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:12,990 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:12,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:12,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:12,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:12,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:12,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:12,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:12,991 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:12,992 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:12,992 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:12,992 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:12,992 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a4dcc1724524b518b661b042db5b670, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1383eab0fa8b4fd8b7287778714bb782, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4111d5ff43f84c128aa0522daa394ad2] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=39.4 K 2024-11-28T01:41:12,993 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4dcc1724524b518b661b042db5b670, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1732758069536 2024-11-28T01:41:12,993 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:12,994 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1383eab0fa8b4fd8b7287778714bb782, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=843, earliestPutTs=1732758070924 2024-11-28T01:41:12,994 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:12,994 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:12,994 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f3d77f604eb048faba82fb04ce1b525f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/eaa5e1224b1d4753bb767a8b345d333d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7873dedf4b544e6bbf54e03f8b46272a] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=37.0 K 2024-11-28T01:41:12,994 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4111d5ff43f84c128aa0522daa394ad2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=862, earliestPutTs=1732758072121 2024-11-28T01:41:12,995 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f3d77f604eb048faba82fb04ce1b525f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1732758069536 2024-11-28T01:41:12,995 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa5e1224b1d4753bb767a8b345d333d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=843, earliestPutTs=1732758070924 2024-11-28T01:41:12,996 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7873dedf4b544e6bbf54e03f8b46272a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=862, earliestPutTs=1732758072121 2024-11-28T01:41:13,013 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:13,014 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/387e794a4e0f44febdeae45bee24883f is 50, key is test_row_0/A:col10/1732758072814/Put/seqid=0 2024-11-28T01:41:13,027 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:13,028 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/28cd97f321c4468ebf6e158e3d7dd57f is 50, key is test_row_0/B:col10/1732758072814/Put/seqid=0 2024-11-28T01:41:13,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741933_1109 (size=13391) 2024-11-28T01:41:13,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741934_1110 (size=13391) 2024-11-28T01:41:13,078 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/28cd97f321c4468ebf6e158e3d7dd57f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/28cd97f321c4468ebf6e158e3d7dd57f 2024-11-28T01:41:13,085 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into 28cd97f321c4468ebf6e158e3d7dd57f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:13,085 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:13,085 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758072991; duration=0sec 2024-11-28T01:41:13,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:13,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:13,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:13,091 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:13,091 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:13,091 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,092 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/940a4ecb7817413aabf787d6a2ea5071, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0e197c4de24045ee95d30d4c31f02cc9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/d0a98de8eb8e4eb4b68f19aaf5135774] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=37.0 K 2024-11-28T01:41:13,094 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 940a4ecb7817413aabf787d6a2ea5071, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=825, earliestPutTs=1732758069536 2024-11-28T01:41:13,094 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e197c4de24045ee95d30d4c31f02cc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=843, earliestPutTs=1732758070924 2024-11-28T01:41:13,095 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a98de8eb8e4eb4b68f19aaf5135774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=862, earliestPutTs=1732758072121 2024-11-28T01:41:13,126 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:13,127 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/de293e22d7434d948ac166c178b5776d is 50, key is test_row_0/C:col10/1732758072814/Put/seqid=0 2024-11-28T01:41:13,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741935_1111 (size=13391) 2024-11-28T01:41:13,170 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/de293e22d7434d948ac166c178b5776d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/de293e22d7434d948ac166c178b5776d 2024-11-28T01:41:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T01:41:13,183 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:13,183 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-28T01:41:13,186 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into de293e22d7434d948ac166c178b5776d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:13,186 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:13,186 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758072991; duration=0sec 2024-11-28T01:41:13,186 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:13,186 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:13,186 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:13,188 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:13,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T01:41:13,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T01:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:13,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:13,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/7a1d85f8a7364f33b4de57c08b260d55 is 50, key is test_row_0/A:col10/1732758073202/Put/seqid=0 2024-11-28T01:41:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758133251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758133253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741936_1112 (size=12301) 2024-11-28T01:41:13,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/7a1d85f8a7364f33b4de57c08b260d55 2024-11-28T01:41:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T01:41:13,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/efbe79284db04d5ca7e98b8a53a75b61 is 50, key is test_row_0/B:col10/1732758073202/Put/seqid=0 2024-11-28T01:41:13,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:13,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:13,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741937_1113 (size=12301) 2024-11-28T01:41:13,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758133363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758133363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,464 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/387e794a4e0f44febdeae45bee24883f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/387e794a4e0f44febdeae45bee24883f 2024-11-28T01:41:13,472 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 387e794a4e0f44febdeae45bee24883f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:13,472 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:13,472 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758072990; duration=0sec 2024-11-28T01:41:13,472 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:13,472 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:13,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T01:41:13,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758133582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758133582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/efbe79284db04d5ca7e98b8a53a75b61 2024-11-28T01:41:13,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7ef40e5a7cae4ea39f07662626e3bc2a is 50, key is test_row_0/C:col10/1732758073202/Put/seqid=0 2024-11-28T01:41:13,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T01:41:13,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741938_1114 (size=12301) 2024-11-28T01:41:13,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7ef40e5a7cae4ea39f07662626e3bc2a 2024-11-28T01:41:13,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/7a1d85f8a7364f33b4de57c08b260d55 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/7a1d85f8a7364f33b4de57c08b260d55 2024-11-28T01:41:13,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/7a1d85f8a7364f33b4de57c08b260d55, entries=150, sequenceid=882, filesize=12.0 K 2024-11-28T01:41:13,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/efbe79284db04d5ca7e98b8a53a75b61 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/efbe79284db04d5ca7e98b8a53a75b61 2024-11-28T01:41:13,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/efbe79284db04d5ca7e98b8a53a75b61, entries=150, sequenceid=882, filesize=12.0 K 2024-11-28T01:41:13,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/7ef40e5a7cae4ea39f07662626e3bc2a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7ef40e5a7cae4ea39f07662626e3bc2a 2024-11-28T01:41:13,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7ef40e5a7cae4ea39f07662626e3bc2a, entries=150, sequenceid=882, filesize=12.0 K 2024-11-28T01:41:13,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 707ae842769f4709bfcd7372850df285 in 675ms, sequenceid=882, compaction requested=false 2024-11-28T01:41:13,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:13,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T01:41:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:13,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/19e1f75f7ece4d25bd2d486bcb63df52 is 50, key is test_row_0/A:col10/1732758073251/Put/seqid=0 2024-11-28T01:41:13,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741939_1115 (size=12301) 2024-11-28T01:41:13,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=899 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/19e1f75f7ece4d25bd2d486bcb63df52 2024-11-28T01:41:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758133953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758133956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:13,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/1a271cf3f7c04504a576b469ebcce159 is 50, key is test_row_0/B:col10/1732758073251/Put/seqid=0 2024-11-28T01:41:13,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:13,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741940_1116 (size=12301) 2024-11-28T01:41:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758134073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758134073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:14,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:14,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758134282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758134282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T01:41:14,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=899 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/1a271cf3f7c04504a576b469ebcce159 2024-11-28T01:41:14,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0264b1bdc65847ebb281b5c63c914298 is 50, key is test_row_0/C:col10/1732758073251/Put/seqid=0 2024-11-28T01:41:14,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741941_1117 (size=12301) 2024-11-28T01:41:14,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:14,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:14,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=899 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0264b1bdc65847ebb281b5c63c914298 2024-11-28T01:41:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:14,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/19e1f75f7ece4d25bd2d486bcb63df52 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/19e1f75f7ece4d25bd2d486bcb63df52 2024-11-28T01:41:14,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/19e1f75f7ece4d25bd2d486bcb63df52, entries=150, sequenceid=899, filesize=12.0 K 2024-11-28T01:41:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/1a271cf3f7c04504a576b469ebcce159 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1a271cf3f7c04504a576b469ebcce159 2024-11-28T01:41:14,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1a271cf3f7c04504a576b469ebcce159, entries=150, sequenceid=899, filesize=12.0 K 2024-11-28T01:41:14,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0264b1bdc65847ebb281b5c63c914298 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0264b1bdc65847ebb281b5c63c914298 2024-11-28T01:41:14,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0264b1bdc65847ebb281b5c63c914298, entries=150, sequenceid=899, filesize=12.0 K 2024-11-28T01:41:14,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 707ae842769f4709bfcd7372850df285 in 577ms, sequenceid=899, compaction requested=true 2024-11-28T01:41:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:14,484 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:14,484 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:14,486 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:14,486 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/A is initiating minor compaction (all files) 2024-11-28T01:41:14,486 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/A in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,486 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/387e794a4e0f44febdeae45bee24883f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/7a1d85f8a7364f33b4de57c08b260d55, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/19e1f75f7ece4d25bd2d486bcb63df52] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=37.1 K 2024-11-28T01:41:14,487 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:14,487 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/B is initiating minor compaction (all files) 2024-11-28T01:41:14,487 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/B in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,487 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/28cd97f321c4468ebf6e158e3d7dd57f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/efbe79284db04d5ca7e98b8a53a75b61, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1a271cf3f7c04504a576b469ebcce159] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=37.1 K 2024-11-28T01:41:14,487 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 387e794a4e0f44febdeae45bee24883f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=862, earliestPutTs=1732758072121 2024-11-28T01:41:14,488 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 28cd97f321c4468ebf6e158e3d7dd57f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=862, earliestPutTs=1732758072121 2024-11-28T01:41:14,488 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a1d85f8a7364f33b4de57c08b260d55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1732758072862 2024-11-28T01:41:14,488 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting efbe79284db04d5ca7e98b8a53a75b61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1732758072862 2024-11-28T01:41:14,489 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e1f75f7ece4d25bd2d486bcb63df52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=899, earliestPutTs=1732758073251 2024-11-28T01:41:14,489 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a271cf3f7c04504a576b469ebcce159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=899, earliestPutTs=1732758073251 2024-11-28T01:41:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:14,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:14,510 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#A#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:14,511 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/25ae7aa9d5c94ee89aa796ea96bf6fa0 is 50, key is test_row_0/A:col10/1732758073251/Put/seqid=0 2024-11-28T01:41:14,521 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:14,521 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/65fdfe85c2364d189cda240009fe9c9b is 50, key is test_row_0/B:col10/1732758073251/Put/seqid=0 2024-11-28T01:41:14,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741942_1118 (size=13493) 2024-11-28T01:41:14,568 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/25ae7aa9d5c94ee89aa796ea96bf6fa0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/25ae7aa9d5c94ee89aa796ea96bf6fa0 2024-11-28T01:41:14,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741943_1119 (size=13493) 2024-11-28T01:41:14,579 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/A of 707ae842769f4709bfcd7372850df285 into 25ae7aa9d5c94ee89aa796ea96bf6fa0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:14,580 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:14,580 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/A, priority=13, startTime=1732758074484; duration=0sec 2024-11-28T01:41:14,580 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:14,580 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:14,580 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:14,581 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/65fdfe85c2364d189cda240009fe9c9b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/65fdfe85c2364d189cda240009fe9c9b 2024-11-28T01:41:14,582 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:14,582 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 707ae842769f4709bfcd7372850df285/C is initiating minor compaction (all files) 2024-11-28T01:41:14,582 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 707ae842769f4709bfcd7372850df285/C in TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,583 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/de293e22d7434d948ac166c178b5776d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7ef40e5a7cae4ea39f07662626e3bc2a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0264b1bdc65847ebb281b5c63c914298] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp, totalSize=37.1 K 2024-11-28T01:41:14,583 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting de293e22d7434d948ac166c178b5776d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=862, earliestPutTs=1732758072121 2024-11-28T01:41:14,585 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ef40e5a7cae4ea39f07662626e3bc2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1732758072862 2024-11-28T01:41:14,586 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0264b1bdc65847ebb281b5c63c914298, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=899, earliestPutTs=1732758073251 2024-11-28T01:41:14,591 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/B of 707ae842769f4709bfcd7372850df285 into 65fdfe85c2364d189cda240009fe9c9b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:14,591 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:14,591 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/B, priority=13, startTime=1732758074484; duration=0sec 2024-11-28T01:41:14,591 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:14,591 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:14,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T01:41:14,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:14,596 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T01:41:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:14,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. as already flushing 2024-11-28T01:41:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:14,603 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 707ae842769f4709bfcd7372850df285#C#compaction#107 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:14,604 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/afeb32a596834ed8b0e7db0a85f32c5d is 50, key is test_row_0/C:col10/1732758073251/Put/seqid=0 2024-11-28T01:41:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/6c443f9ed9374d0d84df9cc9266c3300 is 50, key is test_row_0/A:col10/1732758073951/Put/seqid=0 2024-11-28T01:41:14,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741944_1120 (size=13493) 2024-11-28T01:41:14,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741945_1121 (size=12301) 2024-11-28T01:41:14,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=918 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/6c443f9ed9374d0d84df9cc9266c3300 2024-11-28T01:41:14,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758134646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758134648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,663 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/afeb32a596834ed8b0e7db0a85f32c5d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/afeb32a596834ed8b0e7db0a85f32c5d 2024-11-28T01:41:14,704 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 707ae842769f4709bfcd7372850df285/C of 707ae842769f4709bfcd7372850df285 into afeb32a596834ed8b0e7db0a85f32c5d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:14,704 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:14,704 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., storeName=707ae842769f4709bfcd7372850df285/C, priority=13, startTime=1732758074500; duration=0sec 2024-11-28T01:41:14,705 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:14,705 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:14,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/c6bc1385f19b409ba1613fdd5d8e2d48 is 50, key is test_row_0/B:col10/1732758073951/Put/seqid=0 2024-11-28T01:41:14,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741946_1122 (size=12301) 2024-11-28T01:41:14,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758134762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758134762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41402 deadline: 1732758134972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41336 deadline: 1732758134972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:14,983 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-28T01:41:14,983 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:14,986 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-28T01:41:14,986 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-28T01:41:14,986 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:14,986 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:14,989 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,989 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,989 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,990 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,990 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,990 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:14,990 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:14,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:14,994 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-28T01:41:14,994 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:14,994 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,994 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:14,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:15,125 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=918 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/c6bc1385f19b409ba1613fdd5d8e2d48 2024-11-28T01:41:15,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/037d2ef801644b5f8d22e8612772e2eb is 50, key is test_row_0/C:col10/1732758073951/Put/seqid=0 2024-11-28T01:41:15,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741947_1123 (size=12301) 2024-11-28T01:41:15,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=918 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/037d2ef801644b5f8d22e8612772e2eb 2024-11-28T01:41:15,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/6c443f9ed9374d0d84df9cc9266c3300 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/6c443f9ed9374d0d84df9cc9266c3300 2024-11-28T01:41:15,171 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/6c443f9ed9374d0d84df9cc9266c3300, entries=150, sequenceid=918, filesize=12.0 K 2024-11-28T01:41:15,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/c6bc1385f19b409ba1613fdd5d8e2d48 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/c6bc1385f19b409ba1613fdd5d8e2d48 2024-11-28T01:41:15,181 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/c6bc1385f19b409ba1613fdd5d8e2d48, entries=150, sequenceid=918, filesize=12.0 K 2024-11-28T01:41:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/037d2ef801644b5f8d22e8612772e2eb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/037d2ef801644b5f8d22e8612772e2eb 2024-11-28T01:41:15,188 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/037d2ef801644b5f8d22e8612772e2eb, entries=150, sequenceid=918, filesize=12.0 K 2024-11-28T01:41:15,189 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 707ae842769f4709bfcd7372850df285 in 594ms, sequenceid=918, compaction requested=false 2024-11-28T01:41:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-28T01:41:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-28T01:41:15,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-28T01:41:15,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0030 sec 2024-11-28T01:41:15,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.0110 sec 2024-11-28T01:41:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:15,289 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-28T01:41:15,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T01:41:15,289 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:15,289 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,289 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:15,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:15,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:15,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:15,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:15,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:15,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:15,290 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-28T01:41:15,290 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:15,290 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,291 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:15,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/31f2d6f7e64c426d9cc786c3e8b79a59 is 50, key is test_row_0/A:col10/1732758075283/Put/seqid=0 2024-11-28T01:41:15,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741948_1124 (size=12301) 2024-11-28T01:41:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T01:41:15,343 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:15,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=936 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/31f2d6f7e64c426d9cc786c3e8b79a59 2024-11-28T01:41:15,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/ef5c9004b94a4cc7b3751ba3f0801296 is 50, key is test_row_0/B:col10/1732758075283/Put/seqid=0 2024-11-28T01:41:15,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741949_1125 (size=12301) 2024-11-28T01:41:15,955 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-28T01:41:15,955 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:15,955 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-28T01:41:15,955 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,955 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:15,955 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,955 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,955 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:15,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:15,985 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-28T01:41:15,985 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:15,986 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,986 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 177 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 229 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2272 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2232 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3157 rows 2024-11-28T01:41:15,986 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-11-28T01:41:15,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3181 rows 2024-11-28T01:41:15,987 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:41:15,987 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:41:15,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:15,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:15,992 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T01:41:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T01:41:16,001 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758076000"}]},"ts":"1732758076000"} 2024-11-28T01:41:16,003 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T01:41:16,003 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T01:41:16,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:41:16,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, UNASSIGN}] 2024-11-28T01:41:16,010 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, UNASSIGN 2024-11-28T01:41:16,012 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=707ae842769f4709bfcd7372850df285, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:16,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, UNASSIGN because future has completed 2024-11-28T01:41:16,015 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:41:16,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:41:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T01:41:16,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=936 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/ef5c9004b94a4cc7b3751ba3f0801296 2024-11-28T01:41:16,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0743ec6aa05a47d5b515932995357167 is 50, key is test_row_0/C:col10/1732758075283/Put/seqid=0 2024-11-28T01:41:16,173 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:16,174 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:41:16,174 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 707ae842769f4709bfcd7372850df285, disabling compactions & flushes 2024-11-28T01:41:16,175 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:16,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741950_1126 (size=12301) 2024-11-28T01:41:16,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=936 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0743ec6aa05a47d5b515932995357167 2024-11-28T01:41:16,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/31f2d6f7e64c426d9cc786c3e8b79a59 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/31f2d6f7e64c426d9cc786c3e8b79a59 2024-11-28T01:41:16,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/31f2d6f7e64c426d9cc786c3e8b79a59, entries=150, sequenceid=936, filesize=12.0 K 2024-11-28T01:41:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/ef5c9004b94a4cc7b3751ba3f0801296 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ef5c9004b94a4cc7b3751ba3f0801296 2024-11-28T01:41:16,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ef5c9004b94a4cc7b3751ba3f0801296, entries=150, sequenceid=936, filesize=12.0 K 2024-11-28T01:41:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/0743ec6aa05a47d5b515932995357167 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0743ec6aa05a47d5b515932995357167 2024-11-28T01:41:16,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0743ec6aa05a47d5b515932995357167, entries=150, sequenceid=936, filesize=12.0 K 2024-11-28T01:41:16,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 707ae842769f4709bfcd7372850df285 in 937ms, sequenceid=936, compaction requested=true 2024-11-28T01:41:16,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 707ae842769f4709bfcd7372850df285: 2024-11-28T01:41:16,226 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:16,226 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:16,226 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. after waiting 0 ms 2024-11-28T01:41:16,226 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:16,226 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. because compaction request was cancelled 2024-11-28T01:41:16,226 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 707ae842769f4709bfcd7372850df285 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-28T01:41:16,226 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:A 2024-11-28T01:41:16,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:A, priority=-2147483648, current under compaction store size is 0 2024-11-28T01:41:16,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:16,226 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=A 2024-11-28T01:41:16,226 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. because compaction request was cancelled 2024-11-28T01:41:16,227 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:B 2024-11-28T01:41:16,227 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:B, priority=-2147483648, current under compaction store size is 0 2024-11-28T01:41:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:16,227 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. because compaction request was cancelled 2024-11-28T01:41:16,227 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 707ae842769f4709bfcd7372850df285:C 2024-11-28T01:41:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 707ae842769f4709bfcd7372850df285:C, priority=-2147483648, current under compaction store size is 0 2024-11-28T01:41:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:16,227 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=B 2024-11-28T01:41:16,228 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:16,228 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 707ae842769f4709bfcd7372850df285, store=C 2024-11-28T01:41:16,228 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:16,234 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8dabd64942244a77a5ff8d30e428949b is 50, key is test_row_2/A:col10/1732758075983/Put/seqid=0 2024-11-28T01:41:16,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741951_1127 (size=7415) 2024-11-28T01:41:16,252 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8dabd64942244a77a5ff8d30e428949b 2024-11-28T01:41:16,266 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/3e1556dd4d2047548067fa2229b2bdad is 50, key is test_row_2/B:col10/1732758075983/Put/seqid=0 2024-11-28T01:41:16,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741952_1128 (size=7415) 2024-11-28T01:41:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T01:41:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T01:41:16,681 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/3e1556dd4d2047548067fa2229b2bdad 2024-11-28T01:41:16,690 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/067365a493824288a8abffde25e275c3 is 50, key is test_row_2/C:col10/1732758075983/Put/seqid=0 2024-11-28T01:41:16,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741953_1129 (size=7415) 2024-11-28T01:41:17,096 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=942 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/067365a493824288a8abffde25e275c3 2024-11-28T01:41:17,102 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/A/8dabd64942244a77a5ff8d30e428949b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8dabd64942244a77a5ff8d30e428949b 2024-11-28T01:41:17,106 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8dabd64942244a77a5ff8d30e428949b, entries=50, sequenceid=942, filesize=7.2 K 2024-11-28T01:41:17,107 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/B/3e1556dd4d2047548067fa2229b2bdad as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3e1556dd4d2047548067fa2229b2bdad 2024-11-28T01:41:17,112 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3e1556dd4d2047548067fa2229b2bdad, entries=50, sequenceid=942, filesize=7.2 K 2024-11-28T01:41:17,113 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/.tmp/C/067365a493824288a8abffde25e275c3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/067365a493824288a8abffde25e275c3 2024-11-28T01:41:17,117 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/067365a493824288a8abffde25e275c3, entries=50, sequenceid=942, filesize=7.2 K 2024-11-28T01:41:17,118 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 707ae842769f4709bfcd7372850df285 in 892ms, sequenceid=942, compaction requested=true 2024-11-28T01:41:17,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a3762a22fd7450b8176a14548e9a8e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/e7d76c4930de4172b3830e8e1e50dc2d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/63ef675393804f88a36a56469cb7c582, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ce66cfb4b46f4f23aaf0f51c0138da3d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/36aec1b97d244a5ebc25f8849748df1d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/b4de58042086417a92f08c5aa8ce59fc, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/708feaa7657a4d939de04c27ee5333fd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d51503ae92242c79e13eec3d7566ed2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/744b45ad9efd4f21b383c398c88ee63e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c115e3f3034c4e6b9072b49054d0f117, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/f0caef03c28d4d519fb41a78c007fad2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1117bd21a9a84eaf9b1b1bd6c2b35c07, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ae2ca65929a54bc192960f4bed94e823, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/de2bb5c72f8d460da4387f020a5605cd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/3665f894bc1d48cd89ad94d8e6690fb6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/72f455b93e354d3c8e4735ff895bbf20, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d9b3a974ad94ef18641e162244c5a0d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/d1fe55ec7c3a414f882c404d3ceebb81, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/dd1ffcd4e9e943c2bfd2c4a5b828b384, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4a99ae3a28d74e169f8a833c530a9088, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8e44ccf4cd284240bcfa52c6fb5356b9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ed970bc1238c4894bd69ad3797995179, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/2b751de78cac49598d3a02845f13602c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/90c945b11227414ab4c6f622485121cf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c7d85858da384086b94d16f8a7fdc67a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/840cdc6e12e44853ad49ad69d417c1ee, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/43e73f2d1a2d44afb84d7742d9a47a98, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8cbc6b2b99d949819eac5acf50777f2f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/495a8bdc154a49e49b6d12b0348a3dde, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a4dcc1724524b518b661b042db5b670, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1383eab0fa8b4fd8b7287778714bb782, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4111d5ff43f84c128aa0522daa394ad2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/387e794a4e0f44febdeae45bee24883f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/7a1d85f8a7364f33b4de57c08b260d55, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/19e1f75f7ece4d25bd2d486bcb63df52] to archive 2024-11-28T01:41:17,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:41:17,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a3762a22fd7450b8176a14548e9a8e3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a3762a22fd7450b8176a14548e9a8e3 2024-11-28T01:41:17,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/e7d76c4930de4172b3830e8e1e50dc2d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/e7d76c4930de4172b3830e8e1e50dc2d 2024-11-28T01:41:17,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/63ef675393804f88a36a56469cb7c582 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/63ef675393804f88a36a56469cb7c582 2024-11-28T01:41:17,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ce66cfb4b46f4f23aaf0f51c0138da3d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ce66cfb4b46f4f23aaf0f51c0138da3d 2024-11-28T01:41:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T01:41:17,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/36aec1b97d244a5ebc25f8849748df1d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/36aec1b97d244a5ebc25f8849748df1d 2024-11-28T01:41:17,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/b4de58042086417a92f08c5aa8ce59fc to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/b4de58042086417a92f08c5aa8ce59fc 2024-11-28T01:41:17,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/708feaa7657a4d939de04c27ee5333fd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/708feaa7657a4d939de04c27ee5333fd 2024-11-28T01:41:17,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d51503ae92242c79e13eec3d7566ed2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d51503ae92242c79e13eec3d7566ed2 2024-11-28T01:41:17,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/744b45ad9efd4f21b383c398c88ee63e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/744b45ad9efd4f21b383c398c88ee63e 2024-11-28T01:41:17,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c115e3f3034c4e6b9072b49054d0f117 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c115e3f3034c4e6b9072b49054d0f117 2024-11-28T01:41:17,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/f0caef03c28d4d519fb41a78c007fad2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/f0caef03c28d4d519fb41a78c007fad2 2024-11-28T01:41:17,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1117bd21a9a84eaf9b1b1bd6c2b35c07 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1117bd21a9a84eaf9b1b1bd6c2b35c07 2024-11-28T01:41:17,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ae2ca65929a54bc192960f4bed94e823 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ae2ca65929a54bc192960f4bed94e823 2024-11-28T01:41:17,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/de2bb5c72f8d460da4387f020a5605cd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/de2bb5c72f8d460da4387f020a5605cd 2024-11-28T01:41:17,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/3665f894bc1d48cd89ad94d8e6690fb6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/3665f894bc1d48cd89ad94d8e6690fb6 2024-11-28T01:41:17,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/72f455b93e354d3c8e4735ff895bbf20 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/72f455b93e354d3c8e4735ff895bbf20 2024-11-28T01:41:17,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d9b3a974ad94ef18641e162244c5a0d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/9d9b3a974ad94ef18641e162244c5a0d 2024-11-28T01:41:17,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/d1fe55ec7c3a414f882c404d3ceebb81 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/d1fe55ec7c3a414f882c404d3ceebb81 2024-11-28T01:41:17,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/dd1ffcd4e9e943c2bfd2c4a5b828b384 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/dd1ffcd4e9e943c2bfd2c4a5b828b384 2024-11-28T01:41:17,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4a99ae3a28d74e169f8a833c530a9088 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4a99ae3a28d74e169f8a833c530a9088 2024-11-28T01:41:17,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8e44ccf4cd284240bcfa52c6fb5356b9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8e44ccf4cd284240bcfa52c6fb5356b9 2024-11-28T01:41:17,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ed970bc1238c4894bd69ad3797995179 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/ed970bc1238c4894bd69ad3797995179 2024-11-28T01:41:17,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/2b751de78cac49598d3a02845f13602c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/2b751de78cac49598d3a02845f13602c 2024-11-28T01:41:17,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/90c945b11227414ab4c6f622485121cf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/90c945b11227414ab4c6f622485121cf 2024-11-28T01:41:17,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c7d85858da384086b94d16f8a7fdc67a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/c7d85858da384086b94d16f8a7fdc67a 2024-11-28T01:41:17,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/840cdc6e12e44853ad49ad69d417c1ee to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/840cdc6e12e44853ad49ad69d417c1ee 2024-11-28T01:41:17,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/43e73f2d1a2d44afb84d7742d9a47a98 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/43e73f2d1a2d44afb84d7742d9a47a98 2024-11-28T01:41:17,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8cbc6b2b99d949819eac5acf50777f2f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8cbc6b2b99d949819eac5acf50777f2f 2024-11-28T01:41:17,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/495a8bdc154a49e49b6d12b0348a3dde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/495a8bdc154a49e49b6d12b0348a3dde 2024-11-28T01:41:17,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a4dcc1724524b518b661b042db5b670 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1a4dcc1724524b518b661b042db5b670 2024-11-28T01:41:17,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1383eab0fa8b4fd8b7287778714bb782 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/1383eab0fa8b4fd8b7287778714bb782 2024-11-28T01:41:17,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4111d5ff43f84c128aa0522daa394ad2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/4111d5ff43f84c128aa0522daa394ad2 2024-11-28T01:41:17,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/387e794a4e0f44febdeae45bee24883f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/387e794a4e0f44febdeae45bee24883f 2024-11-28T01:41:17,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/7a1d85f8a7364f33b4de57c08b260d55 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/7a1d85f8a7364f33b4de57c08b260d55 2024-11-28T01:41:17,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/19e1f75f7ece4d25bd2d486bcb63df52 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/19e1f75f7ece4d25bd2d486bcb63df52 2024-11-28T01:41:17,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f63dbe6fdc334c9582712f41f03431df, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4c8a684de3e64828a5d71a2fdc6c699b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/bf496e0df75e4b4ba536311bbea86f8f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/57eef107197e41a5a84818a5ef1da2af, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/738330cb98664ddaa45a43a5ba620253, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d693de301e40487fa787d499fd4c31f6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/86bff3ebe90f46d09bc8a2fd5109c18b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ca40c2170aae4b618776fa865d63b5cf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f2ac74457b2f4b17ad169b5434ee9cfe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/dcc0eec9f71e47f797924a1b2621228b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4cda20d8e54a4d61a311a22b4b190739, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f5512650135543dba9db35448de7ef0b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/91914a43f7c9432dac65590c786e0153, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7b9724134dc7409ea3bc4781255371ec, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/997e283f1dba4c0f81ffc07b570f63d7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1ca4c6a5b4994b6c87ae0657fb26d518, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/0700e1ab4a894df8870e752405055362, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d45a56d444c3469c99b95cf750acfafd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7821ef34a2674a769d3b3ae65afee4ad, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/371e0d67a63b43ee84cf54c3a93405b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/a953b083226a45b4bfd85b97e8ceec4f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3edc8593bd7e43a39842f2bdc42bd93d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f4537fceddef43faaba0e5f6a729641b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/68792f3d2756425baff7a7e8a94e8d2e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/9b127700ea774ef09d88a6273d8697b5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/2836ac874c434a428b69bb1ae8d1a772, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d67156137d6247198d68a9b82d6d893d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d8ff34194e324d688dfdbec6f6d42c5a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f3d77f604eb048faba82fb04ce1b525f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4fa8b86f80ed4c99930893b787cbe3d4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/eaa5e1224b1d4753bb767a8b345d333d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/28cd97f321c4468ebf6e158e3d7dd57f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7873dedf4b544e6bbf54e03f8b46272a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/efbe79284db04d5ca7e98b8a53a75b61, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1a271cf3f7c04504a576b469ebcce159] to archive 2024-11-28T01:41:17,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:41:17,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f63dbe6fdc334c9582712f41f03431df to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f63dbe6fdc334c9582712f41f03431df 2024-11-28T01:41:17,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4c8a684de3e64828a5d71a2fdc6c699b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4c8a684de3e64828a5d71a2fdc6c699b 2024-11-28T01:41:17,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/bf496e0df75e4b4ba536311bbea86f8f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/bf496e0df75e4b4ba536311bbea86f8f 2024-11-28T01:41:17,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/57eef107197e41a5a84818a5ef1da2af to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/57eef107197e41a5a84818a5ef1da2af 2024-11-28T01:41:17,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/738330cb98664ddaa45a43a5ba620253 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/738330cb98664ddaa45a43a5ba620253 2024-11-28T01:41:17,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d693de301e40487fa787d499fd4c31f6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d693de301e40487fa787d499fd4c31f6 2024-11-28T01:41:17,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/86bff3ebe90f46d09bc8a2fd5109c18b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/86bff3ebe90f46d09bc8a2fd5109c18b 2024-11-28T01:41:17,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ca40c2170aae4b618776fa865d63b5cf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ca40c2170aae4b618776fa865d63b5cf 2024-11-28T01:41:17,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f2ac74457b2f4b17ad169b5434ee9cfe to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f2ac74457b2f4b17ad169b5434ee9cfe 2024-11-28T01:41:17,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/dcc0eec9f71e47f797924a1b2621228b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/dcc0eec9f71e47f797924a1b2621228b 2024-11-28T01:41:17,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4cda20d8e54a4d61a311a22b4b190739 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4cda20d8e54a4d61a311a22b4b190739 2024-11-28T01:41:17,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f5512650135543dba9db35448de7ef0b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f5512650135543dba9db35448de7ef0b 2024-11-28T01:41:17,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/91914a43f7c9432dac65590c786e0153 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/91914a43f7c9432dac65590c786e0153 2024-11-28T01:41:17,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7b9724134dc7409ea3bc4781255371ec to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7b9724134dc7409ea3bc4781255371ec 2024-11-28T01:41:17,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/997e283f1dba4c0f81ffc07b570f63d7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/997e283f1dba4c0f81ffc07b570f63d7 2024-11-28T01:41:17,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1ca4c6a5b4994b6c87ae0657fb26d518 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1ca4c6a5b4994b6c87ae0657fb26d518 2024-11-28T01:41:17,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/0700e1ab4a894df8870e752405055362 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/0700e1ab4a894df8870e752405055362 2024-11-28T01:41:17,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d45a56d444c3469c99b95cf750acfafd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d45a56d444c3469c99b95cf750acfafd 2024-11-28T01:41:17,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7821ef34a2674a769d3b3ae65afee4ad to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7821ef34a2674a769d3b3ae65afee4ad 2024-11-28T01:41:17,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/371e0d67a63b43ee84cf54c3a93405b1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/371e0d67a63b43ee84cf54c3a93405b1 2024-11-28T01:41:17,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/a953b083226a45b4bfd85b97e8ceec4f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/a953b083226a45b4bfd85b97e8ceec4f 2024-11-28T01:41:17,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3edc8593bd7e43a39842f2bdc42bd93d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3edc8593bd7e43a39842f2bdc42bd93d 2024-11-28T01:41:17,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f4537fceddef43faaba0e5f6a729641b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f4537fceddef43faaba0e5f6a729641b 2024-11-28T01:41:17,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/68792f3d2756425baff7a7e8a94e8d2e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/68792f3d2756425baff7a7e8a94e8d2e 2024-11-28T01:41:17,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/9b127700ea774ef09d88a6273d8697b5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/9b127700ea774ef09d88a6273d8697b5 2024-11-28T01:41:17,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/2836ac874c434a428b69bb1ae8d1a772 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/2836ac874c434a428b69bb1ae8d1a772 2024-11-28T01:41:17,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d67156137d6247198d68a9b82d6d893d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d67156137d6247198d68a9b82d6d893d 2024-11-28T01:41:17,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d8ff34194e324d688dfdbec6f6d42c5a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/d8ff34194e324d688dfdbec6f6d42c5a 2024-11-28T01:41:17,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f3d77f604eb048faba82fb04ce1b525f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/f3d77f604eb048faba82fb04ce1b525f 2024-11-28T01:41:17,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4fa8b86f80ed4c99930893b787cbe3d4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/4fa8b86f80ed4c99930893b787cbe3d4 2024-11-28T01:41:17,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/eaa5e1224b1d4753bb767a8b345d333d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/eaa5e1224b1d4753bb767a8b345d333d 2024-11-28T01:41:17,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/28cd97f321c4468ebf6e158e3d7dd57f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/28cd97f321c4468ebf6e158e3d7dd57f 2024-11-28T01:41:17,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7873dedf4b544e6bbf54e03f8b46272a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/7873dedf4b544e6bbf54e03f8b46272a 2024-11-28T01:41:17,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/efbe79284db04d5ca7e98b8a53a75b61 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/efbe79284db04d5ca7e98b8a53a75b61 2024-11-28T01:41:17,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1a271cf3f7c04504a576b469ebcce159 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/1a271cf3f7c04504a576b469ebcce159 2024-11-28T01:41:17,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b5d92fc43c5347008d669898688ebc4b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/36c8f77911714f258e4852452ce1b504, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/fb310e9761ab4a2b90441777bf31bfa6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0bdecff9fe69418c944d6493723793fe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/11abecd61a5c4bfdbfae5882cce7a424, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/06b6bb5258c44a459bb0aa1d13eb5266, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a043dc5b2c7d4696817aad2c5e1d43b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/4a72e13870074460bc210727affc7176, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/1fdeb1377646414cae643d159f63aad0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bab6ed42d5d74b7aaffeba8c2f32546b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e9d945b974724d3aa58482a0d5356068, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b64e33236c8e4034917810d12318d0fb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/031278f9365e4b3ea610501083d3600c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7bc04cfd2cb74045ba52369721463ede, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/5585104e81344c548872da8d261565cf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/96e58dc3e9ff4909aafebe640e63e6a5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a15fe028fb9446a9889023a14faebde4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7461f245e0cb4eb1af107141c4a53997, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/9561c729e1614c6e87663801f4983d84, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e69b650dfa7b4a819aca050070ce9ae2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6109f550e70940a39918b81d9e0dd8ce, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a47ec7c838d449c3933f7259de8980f2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/ae54f3dc30d54867863a690e4af37eae, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f55b180236da42c98039c8a7c9496445, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bfff018a884e48cd9fa6a7e2d917d6a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f24ba7cb18e14da2b81903f4048afecb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/c0e3d9db1e2d4a3cb808e8496f22410c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6c0dc0223ea24e34bd67720fbf92a95e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/940a4ecb7817413aabf787d6a2ea5071, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/83951d4a077e464f81b264db3f65eb43, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0e197c4de24045ee95d30d4c31f02cc9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/de293e22d7434d948ac166c178b5776d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/d0a98de8eb8e4eb4b68f19aaf5135774, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7ef40e5a7cae4ea39f07662626e3bc2a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0264b1bdc65847ebb281b5c63c914298] to archive 2024-11-28T01:41:17,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:41:17,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b5d92fc43c5347008d669898688ebc4b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b5d92fc43c5347008d669898688ebc4b 2024-11-28T01:41:17,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/36c8f77911714f258e4852452ce1b504 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/36c8f77911714f258e4852452ce1b504 2024-11-28T01:41:17,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/fb310e9761ab4a2b90441777bf31bfa6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/fb310e9761ab4a2b90441777bf31bfa6 2024-11-28T01:41:17,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0bdecff9fe69418c944d6493723793fe to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0bdecff9fe69418c944d6493723793fe 2024-11-28T01:41:17,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/11abecd61a5c4bfdbfae5882cce7a424 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/11abecd61a5c4bfdbfae5882cce7a424 2024-11-28T01:41:17,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/06b6bb5258c44a459bb0aa1d13eb5266 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/06b6bb5258c44a459bb0aa1d13eb5266 2024-11-28T01:41:17,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a043dc5b2c7d4696817aad2c5e1d43b1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a043dc5b2c7d4696817aad2c5e1d43b1 2024-11-28T01:41:17,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/4a72e13870074460bc210727affc7176 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/4a72e13870074460bc210727affc7176 2024-11-28T01:41:17,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/1fdeb1377646414cae643d159f63aad0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/1fdeb1377646414cae643d159f63aad0 2024-11-28T01:41:17,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bab6ed42d5d74b7aaffeba8c2f32546b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bab6ed42d5d74b7aaffeba8c2f32546b 2024-11-28T01:41:17,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e9d945b974724d3aa58482a0d5356068 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e9d945b974724d3aa58482a0d5356068 2024-11-28T01:41:17,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b64e33236c8e4034917810d12318d0fb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/b64e33236c8e4034917810d12318d0fb 2024-11-28T01:41:17,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/031278f9365e4b3ea610501083d3600c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/031278f9365e4b3ea610501083d3600c 2024-11-28T01:41:17,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7bc04cfd2cb74045ba52369721463ede to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7bc04cfd2cb74045ba52369721463ede 2024-11-28T01:41:17,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/5585104e81344c548872da8d261565cf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/5585104e81344c548872da8d261565cf 2024-11-28T01:41:17,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/96e58dc3e9ff4909aafebe640e63e6a5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/96e58dc3e9ff4909aafebe640e63e6a5 2024-11-28T01:41:17,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a15fe028fb9446a9889023a14faebde4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a15fe028fb9446a9889023a14faebde4 2024-11-28T01:41:17,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7461f245e0cb4eb1af107141c4a53997 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7461f245e0cb4eb1af107141c4a53997 2024-11-28T01:41:17,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/9561c729e1614c6e87663801f4983d84 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/9561c729e1614c6e87663801f4983d84 2024-11-28T01:41:17,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e69b650dfa7b4a819aca050070ce9ae2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/e69b650dfa7b4a819aca050070ce9ae2 2024-11-28T01:41:17,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6109f550e70940a39918b81d9e0dd8ce to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6109f550e70940a39918b81d9e0dd8ce 2024-11-28T01:41:17,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a47ec7c838d449c3933f7259de8980f2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/a47ec7c838d449c3933f7259de8980f2 2024-11-28T01:41:17,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/ae54f3dc30d54867863a690e4af37eae to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/ae54f3dc30d54867863a690e4af37eae 2024-11-28T01:41:17,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f55b180236da42c98039c8a7c9496445 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f55b180236da42c98039c8a7c9496445 2024-11-28T01:41:17,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bfff018a884e48cd9fa6a7e2d917d6a1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/bfff018a884e48cd9fa6a7e2d917d6a1 2024-11-28T01:41:17,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f24ba7cb18e14da2b81903f4048afecb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/f24ba7cb18e14da2b81903f4048afecb 2024-11-28T01:41:17,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/c0e3d9db1e2d4a3cb808e8496f22410c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/c0e3d9db1e2d4a3cb808e8496f22410c 2024-11-28T01:41:17,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6c0dc0223ea24e34bd67720fbf92a95e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/6c0dc0223ea24e34bd67720fbf92a95e 2024-11-28T01:41:17,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/940a4ecb7817413aabf787d6a2ea5071 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/940a4ecb7817413aabf787d6a2ea5071 2024-11-28T01:41:17,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/83951d4a077e464f81b264db3f65eb43 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/83951d4a077e464f81b264db3f65eb43 2024-11-28T01:41:17,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0e197c4de24045ee95d30d4c31f02cc9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0e197c4de24045ee95d30d4c31f02cc9 2024-11-28T01:41:17,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/de293e22d7434d948ac166c178b5776d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/de293e22d7434d948ac166c178b5776d 2024-11-28T01:41:17,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/d0a98de8eb8e4eb4b68f19aaf5135774 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/d0a98de8eb8e4eb4b68f19aaf5135774 2024-11-28T01:41:17,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7ef40e5a7cae4ea39f07662626e3bc2a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/7ef40e5a7cae4ea39f07662626e3bc2a 2024-11-28T01:41:17,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0264b1bdc65847ebb281b5c63c914298 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0264b1bdc65847ebb281b5c63c914298 2024-11-28T01:41:17,311 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/recovered.edits/945.seqid, newMaxSeqId=945, maxSeqId=1 2024-11-28T01:41:17,313 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. 2024-11-28T01:41:17,313 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 707ae842769f4709bfcd7372850df285: Waiting for close lock at 1732758076174Running coprocessor pre-close hooks at 1732758076174Disabling compacts and flushes for region at 1732758076174Disabling writes for close at 1732758076226 (+52 ms)Obtaining lock to block concurrent updates at 1732758076226Preparing flush snapshotting stores in 707ae842769f4709bfcd7372850df285 at 1732758076226Finished memstore snapshotting TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732758076228 (+2 ms)Flushing stores of TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285. at 1732758076228Flushing 707ae842769f4709bfcd7372850df285/A: creating writer at 1732758076228Flushing 707ae842769f4709bfcd7372850df285/A: appending metadata at 1732758076233 (+5 ms)Flushing 707ae842769f4709bfcd7372850df285/A: closing flushed file at 1732758076233Flushing 707ae842769f4709bfcd7372850df285/B: creating writer at 1732758076259 (+26 ms)Flushing 707ae842769f4709bfcd7372850df285/B: appending metadata at 1732758076265 (+6 ms)Flushing 707ae842769f4709bfcd7372850df285/B: closing flushed file at 1732758076265Flushing 707ae842769f4709bfcd7372850df285/C: creating writer at 1732758076687 (+422 ms)Flushing 707ae842769f4709bfcd7372850df285/C: appending metadata at 1732758076690 (+3 ms)Flushing 707ae842769f4709bfcd7372850df285/C: closing flushed file at 1732758076690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6edd5844: reopening flushed file at 1732758077101 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ab26902: reopening flushed file at 1732758077106 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e9d1da0: reopening flushed file at 1732758077112 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 707ae842769f4709bfcd7372850df285 in 892ms, sequenceid=942, compaction requested=true at 1732758077118 (+6 ms)Writing region close event to WAL at 1732758077307 (+189 ms)Running coprocessor post-close hooks at 1732758077312 (+5 ms)Closed at 1732758077312 2024-11-28T01:41:17,316 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 707ae842769f4709bfcd7372850df285 2024-11-28T01:41:17,316 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=707ae842769f4709bfcd7372850df285, regionState=CLOSED 2024-11-28T01:41:17,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:41:17,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-28T01:41:17,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 707ae842769f4709bfcd7372850df285, server=9df3f25f5867,42371,1732758050017 in 1.3030 sec 2024-11-28T01:41:17,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-28T01:41:17,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=707ae842769f4709bfcd7372850df285, UNASSIGN in 1.3120 sec 2024-11-28T01:41:17,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T01:41:17,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3200 sec 2024-11-28T01:41:17,328 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758077327"}]},"ts":"1732758077327"} 2024-11-28T01:41:17,329 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T01:41:17,329 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T01:41:17,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3360 sec 2024-11-28T01:41:18,019 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T01:41:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T01:41:18,143 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:18,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T01:41:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:18,154 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:18,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T01:41:18,172 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285 2024-11-28T01:41:18,196 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/recovered.edits] 2024-11-28T01:41:18,202 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/25ae7aa9d5c94ee89aa796ea96bf6fa0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/25ae7aa9d5c94ee89aa796ea96bf6fa0 2024-11-28T01:41:18,204 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/31f2d6f7e64c426d9cc786c3e8b79a59 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/31f2d6f7e64c426d9cc786c3e8b79a59 2024-11-28T01:41:18,205 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/6c443f9ed9374d0d84df9cc9266c3300 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/6c443f9ed9374d0d84df9cc9266c3300 2024-11-28T01:41:18,207 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8dabd64942244a77a5ff8d30e428949b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/A/8dabd64942244a77a5ff8d30e428949b 2024-11-28T01:41:18,211 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3e1556dd4d2047548067fa2229b2bdad to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/3e1556dd4d2047548067fa2229b2bdad 2024-11-28T01:41:18,213 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/65fdfe85c2364d189cda240009fe9c9b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/65fdfe85c2364d189cda240009fe9c9b 2024-11-28T01:41:18,215 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/c6bc1385f19b409ba1613fdd5d8e2d48 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/c6bc1385f19b409ba1613fdd5d8e2d48 2024-11-28T01:41:18,217 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ef5c9004b94a4cc7b3751ba3f0801296 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/B/ef5c9004b94a4cc7b3751ba3f0801296 2024-11-28T01:41:18,220 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/037d2ef801644b5f8d22e8612772e2eb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/037d2ef801644b5f8d22e8612772e2eb 2024-11-28T01:41:18,222 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/067365a493824288a8abffde25e275c3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/067365a493824288a8abffde25e275c3 2024-11-28T01:41:18,224 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0743ec6aa05a47d5b515932995357167 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/0743ec6aa05a47d5b515932995357167 2024-11-28T01:41:18,227 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/afeb32a596834ed8b0e7db0a85f32c5d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/C/afeb32a596834ed8b0e7db0a85f32c5d 2024-11-28T01:41:18,234 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/recovered.edits/945.seqid to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285/recovered.edits/945.seqid 2024-11-28T01:41:18,239 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/707ae842769f4709bfcd7372850df285 2024-11-28T01:41:18,239 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T01:41:18,262 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T01:41:18,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-28T01:41:18,275 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T01:41:18,322 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T01:41:18,324 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:18,324 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T01:41:18,325 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732758078324"}]},"ts":"9223372036854775807"} 2024-11-28T01:41:18,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T01:41:18,331 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 707ae842769f4709bfcd7372850df285, NAME => 'TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T01:41:18,331 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T01:41:18,332 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732758078331"}]},"ts":"9223372036854775807"} 2024-11-28T01:41:18,340 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T01:41:18,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:18,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 193 msec 2024-11-28T01:41:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T01:41:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T01:41:18,474 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:18,491 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=246 (was 209) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_752253615_22 at /127.0.0.1:38930 [Waiting for operation #364] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9df3f25f5867:42371-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1056209077_22 at /127.0.0.1:47520 [Waiting for operation #369] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=545 (was 429) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7536 (was 7910) 2024-11-28T01:41:18,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=545, ProcessCount=11, AvailableMemoryMB=7535 2024-11-28T01:41:18,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:41:18,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:41:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:18,513 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:41:18,513 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:18,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-28T01:41:18,514 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:41:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T01:41:18,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741954_1130 (size=1061) 2024-11-28T01:41:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T01:41:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T01:41:18,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:41:18,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741955_1131 (size=53) 2024-11-28T01:41:18,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:41:18,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 95ac6ec43a1457c3aa9b4726f69c1394, disabling compactions & flushes 2024-11-28T01:41:18,957 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:18,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:18,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. after waiting 0 ms 2024-11-28T01:41:18,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:18,957 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:18,957 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 95ac6ec43a1457c3aa9b4726f69c1394: Waiting for close lock at 1732758078957Disabling compacts and flushes for region at 1732758078957Disabling writes for close at 1732758078957Writing region close event to WAL at 1732758078957Closed at 1732758078957 2024-11-28T01:41:18,958 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:41:18,959 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732758078958"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758078958"}]},"ts":"1732758078958"} 2024-11-28T01:41:18,960 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:41:18,961 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:41:18,961 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758078961"}]},"ts":"1732758078961"} 2024-11-28T01:41:18,963 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T01:41:18,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, ASSIGN}] 2024-11-28T01:41:18,964 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, ASSIGN 2024-11-28T01:41:18,965 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, ASSIGN; state=OFFLINE, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=false 2024-11-28T01:41:19,116 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:19,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, ASSIGN because future has completed 2024-11-28T01:41:19,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:41:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T01:41:19,276 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:19,276 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:41:19,276 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,276 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:41:19,277 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,277 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,278 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,279 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:19,281 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95ac6ec43a1457c3aa9b4726f69c1394 columnFamilyName A 2024-11-28T01:41:19,281 DEBUG [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:19,282 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(327): Store=95ac6ec43a1457c3aa9b4726f69c1394/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:19,282 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,283 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:19,283 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95ac6ec43a1457c3aa9b4726f69c1394 columnFamilyName B 2024-11-28T01:41:19,283 DEBUG [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:19,284 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(327): Store=95ac6ec43a1457c3aa9b4726f69c1394/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:19,284 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,285 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:19,285 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95ac6ec43a1457c3aa9b4726f69c1394 columnFamilyName C 2024-11-28T01:41:19,285 DEBUG [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:19,286 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(327): Store=95ac6ec43a1457c3aa9b4726f69c1394/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:19,286 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:19,286 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,287 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,287 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,289 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,289 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,289 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:41:19,290 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,292 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:41:19,293 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 95ac6ec43a1457c3aa9b4726f69c1394; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75419037, jitterRate=0.12383122742176056}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:41:19,293 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:19,294 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 95ac6ec43a1457c3aa9b4726f69c1394: Running coprocessor pre-open hook at 1732758079277Writing region info on filesystem at 1732758079277Initializing all the Stores at 1732758079278 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758079278Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758079278Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758079278Cleaning up temporary data from old regions at 1732758079289 (+11 ms)Running coprocessor post-open hooks at 1732758079293 (+4 ms)Region opened successfully at 1732758079293 2024-11-28T01:41:19,294 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., pid=32, masterSystemTime=1732758079272 2024-11-28T01:41:19,296 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:19,296 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:19,297 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:19,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:41:19,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-28T01:41:19,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 in 181 msec 2024-11-28T01:41:19,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-28T01:41:19,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, ASSIGN in 340 msec 2024-11-28T01:41:19,307 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:41:19,307 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758079307"}]},"ts":"1732758079307"} 2024-11-28T01:41:19,309 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T01:41:19,310 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:41:19,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 801 msec 2024-11-28T01:41:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-28T01:41:19,653 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:19,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e98635d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:19,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:19,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:19,658 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b63aa39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:19,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:19,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:19,664 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aafdf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:19,669 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:19,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:19,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:19,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:19,674 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:41:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,45903,1732758049085 2024-11-28T01:41:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c1b1db5 2024-11-28T01:41:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:41:19,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:41:19,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:41:19,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:41:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:19,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741956_1132 (size=1097) 2024-11-28T01:41:20,125 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T01:41:20,125 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T01:41:20,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:41:20,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, REOPEN/MOVE}] 2024-11-28T01:41:20,136 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, REOPEN/MOVE 2024-11-28T01:41:20,137 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, REOPEN/MOVE because future has completed 2024-11-28T01:41:20,139 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:41:20,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:41:20,292 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,292 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:41:20,292 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 95ac6ec43a1457c3aa9b4726f69c1394, disabling compactions & flushes 2024-11-28T01:41:20,292 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,292 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,292 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. after waiting 0 ms 2024-11-28T01:41:20,292 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,296 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T01:41:20,297 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,297 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 95ac6ec43a1457c3aa9b4726f69c1394: Waiting for close lock at 1732758080292Running coprocessor pre-close hooks at 1732758080292Disabling compacts and flushes for region at 1732758080292Disabling writes for close at 1732758080292Writing region close event to WAL at 1732758080293 (+1 ms)Running coprocessor post-close hooks at 1732758080297 (+4 ms)Closed at 1732758080297 2024-11-28T01:41:20,297 WARN [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 95ac6ec43a1457c3aa9b4726f69c1394 to self. 2024-11-28T01:41:20,299 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=CLOSED 2024-11-28T01:41:20,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:41:20,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-28T01:41:20,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 in 163 msec 2024-11-28T01:41:20,305 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, REOPEN/MOVE; state=CLOSED, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=true 2024-11-28T01:41:20,455 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, REOPEN/MOVE because future has completed 2024-11-28T01:41:20,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:41:20,617 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,617 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:41:20,618 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,618 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:41:20,618 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,618 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,622 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,623 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:20,630 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95ac6ec43a1457c3aa9b4726f69c1394 columnFamilyName A 2024-11-28T01:41:20,632 DEBUG [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:20,633 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(327): Store=95ac6ec43a1457c3aa9b4726f69c1394/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:20,634 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,635 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:20,635 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95ac6ec43a1457c3aa9b4726f69c1394 columnFamilyName B 2024-11-28T01:41:20,635 DEBUG [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:20,635 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(327): Store=95ac6ec43a1457c3aa9b4726f69c1394/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:20,636 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,636 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:20,637 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95ac6ec43a1457c3aa9b4726f69c1394 columnFamilyName C 2024-11-28T01:41:20,637 DEBUG [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:20,637 INFO [StoreOpener-95ac6ec43a1457c3aa9b4726f69c1394-1 {}] regionserver.HStore(327): Store=95ac6ec43a1457c3aa9b4726f69c1394/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:20,637 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,637 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,638 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,640 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,641 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,641 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,642 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:41:20,643 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,645 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 95ac6ec43a1457c3aa9b4726f69c1394; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60871933, jitterRate=-0.09293751418590546}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:41:20,645 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,646 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 95ac6ec43a1457c3aa9b4726f69c1394: Running coprocessor pre-open hook at 1732758080618Writing region info on filesystem at 1732758080618Initializing all the Stores at 1732758080619 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758080620 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758080620Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758080620Cleaning up temporary data from old regions at 1732758080641 (+21 ms)Running coprocessor post-open hooks at 1732758080645 (+4 ms)Region opened successfully at 1732758080646 (+1 ms) 2024-11-28T01:41:20,647 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., pid=37, masterSystemTime=1732758080614 2024-11-28T01:41:20,649 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,649 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,650 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=OPEN, openSeqNum=5, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:41:20,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-28T01:41:20,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 in 195 msec 2024-11-28T01:41:20,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-28T01:41:20,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, REOPEN/MOVE in 520 msec 2024-11-28T01:41:20,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-28T01:41:20,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-28T01:41:20,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 976 msec 2024-11-28T01:41:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-28T01:41:20,669 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:20,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec534c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,672 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@475c081c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1045d7f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263f8ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,685 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30213712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,687 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633d822e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c680a91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2298b013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,696 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d93db6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,698 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28367040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,701 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bde3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,703 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc0ff9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,709 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6374ad2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc4bcb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d80d8ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,720 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2441aae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,725 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a9f068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1266e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:20,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:20,734 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceaf575, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:20,736 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7278e272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:20,743 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,745 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,746 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,747 DEBUG [Thread-669 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,747 DEBUG [Thread-669 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,748 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,752 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,772 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,772 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-28T01:41:20,777 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:20,778 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:20,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:20,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T01:41:20,784 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,787 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:20,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:41:20,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732758080819, max timestamp=1732758080839Num uniques 859; 2024-11-28T01:41:20,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758080819, max timestamp=1732758080839Num uniques 859; 2024-11-28T01:41:20,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758080819, max timestamp=1732758080839Num uniques 859; 2024-11-28T01:41:20,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:20,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:20,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287b5ecdcd4c78449abd5d2d81a21c8061_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758080771/Put/seqid=0 2024-11-28T01:41:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T01:41:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758140883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758140887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:20,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758140893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:20,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758140893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:20,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758140914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:20,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:20,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T01:41:20,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:20,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:20,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:20,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741957_1133 (size=12154) 2024-11-28T01:41:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:20,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:20,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287b5ecdcd4c78449abd5d2d81a21c8061_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287b5ecdcd4c78449abd5d2d81a21c8061_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:20,944 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/bc8dea9c0b5e44df97119cba59e54cfa, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:20,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/bc8dea9c0b5e44df97119cba59e54cfa is 175, key is test_row_0/A:col10/1732758080771/Put/seqid=0 2024-11-28T01:41:20,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741958_1134 (size=30955) 2024-11-28T01:41:21,000 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/bc8dea9c0b5e44df97119cba59e54cfa 2024-11-28T01:41:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758141022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758141023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758141023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758141028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758141033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/0952565ebb354cd2bb3f9078dc3c6b1e is 50, key is test_row_0/B:col10/1732758080771/Put/seqid=0 2024-11-28T01:41:21,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741959_1135 (size=12001) 2024-11-28T01:41:21,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T01:41:21,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:21,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T01:41:21,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758141233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T01:41:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758141233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758141233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758141252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758141252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T01:41:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T01:41:21,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/0952565ebb354cd2bb3f9078dc3c6b1e 2024-11-28T01:41:21,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T01:41:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:21,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/d50333073f514ee0ac85522f616b4340 is 50, key is test_row_0/C:col10/1732758080771/Put/seqid=0 2024-11-28T01:41:21,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758141553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758141553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758141553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758141572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758141572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:21,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741960_1136 (size=12001) 2024-11-28T01:41:21,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/d50333073f514ee0ac85522f616b4340 2024-11-28T01:41:21,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/bc8dea9c0b5e44df97119cba59e54cfa as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/bc8dea9c0b5e44df97119cba59e54cfa 2024-11-28T01:41:21,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/bc8dea9c0b5e44df97119cba59e54cfa, entries=150, sequenceid=15, filesize=30.2 K 2024-11-28T01:41:21,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/0952565ebb354cd2bb3f9078dc3c6b1e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/0952565ebb354cd2bb3f9078dc3c6b1e 2024-11-28T01:41:21,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/0952565ebb354cd2bb3f9078dc3c6b1e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T01:41:21,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/d50333073f514ee0ac85522f616b4340 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d50333073f514ee0ac85522f616b4340 2024-11-28T01:41:21,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d50333073f514ee0ac85522f616b4340, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T01:41:21,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 95ac6ec43a1457c3aa9b4726f69c1394 in 875ms, sequenceid=15, compaction requested=false 2024-11-28T01:41:21,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:21,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-28T01:41:21,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:21,708 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-28T01:41:21,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:21,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:21,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:21,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:21,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:21,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:21,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a1c6c555b9cc4cf29e7e861b91ecc536_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758080900/Put/seqid=0 2024-11-28T01:41:21,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741961_1137 (size=12154) 2024-11-28T01:41:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T01:41:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:22,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:22,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758142098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758142099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758142101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758142102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758142104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:22,200 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a1c6c555b9cc4cf29e7e861b91ecc536_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a1c6c555b9cc4cf29e7e861b91ecc536_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/f51463a53ff8427c8478b695aebc3633, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:22,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/f51463a53ff8427c8478b695aebc3633 is 175, key is test_row_0/A:col10/1732758080900/Put/seqid=0 2024-11-28T01:41:22,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758142212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758142212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758142223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758142223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758142223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741962_1138 (size=30955) 2024-11-28T01:41:22,236 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/f51463a53ff8427c8478b695aebc3633 2024-11-28T01:41:22,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/593138f401bf42e9aa33d95d0c36604c is 50, key is test_row_0/B:col10/1732758080900/Put/seqid=0 2024-11-28T01:41:22,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741963_1139 (size=12001) 2024-11-28T01:41:22,317 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/593138f401bf42e9aa33d95d0c36604c 2024-11-28T01:41:22,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/d0e19489ea1f4431b0106c565708ca97 is 50, key is test_row_0/C:col10/1732758080900/Put/seqid=0 2024-11-28T01:41:22,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741964_1140 (size=12001) 2024-11-28T01:41:22,391 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/d0e19489ea1f4431b0106c565708ca97 2024-11-28T01:41:22,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/f51463a53ff8427c8478b695aebc3633 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/f51463a53ff8427c8478b695aebc3633 2024-11-28T01:41:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758142422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758142424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,425 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/f51463a53ff8427c8478b695aebc3633, entries=150, sequenceid=49, filesize=30.2 K 2024-11-28T01:41:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/593138f401bf42e9aa33d95d0c36604c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/593138f401bf42e9aa33d95d0c36604c 2024-11-28T01:41:22,434 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/593138f401bf42e9aa33d95d0c36604c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T01:41:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758142433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/d0e19489ea1f4431b0106c565708ca97 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d0e19489ea1f4431b0106c565708ca97 2024-11-28T01:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,443 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d0e19489ea1f4431b0106c565708ca97, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T01:41:22,445 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 95ac6ec43a1457c3aa9b4726f69c1394 in 737ms, sequenceid=49, compaction requested=false 2024-11-28T01:41:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-28T01:41:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-28T01:41:22,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-28T01:41:22,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6690 sec 2024-11-28T01:41:22,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:41:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:22,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6810 sec 2024-11-28T01:41:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:22,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:22,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ff946e873ef44f42af1741cba065c680_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758082450/Put/seqid=0 2024-11-28T01:41:22,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741965_1141 (size=12154) 2024-11-28T01:41:22,531 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:22,537 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ff946e873ef44f42af1741cba065c680_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ff946e873ef44f42af1741cba065c680_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:22,541 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ddf25d33241f45cfaf8ae98f3dd4952f, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:22,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ddf25d33241f45cfaf8ae98f3dd4952f is 175, key is test_row_0/A:col10/1732758082450/Put/seqid=0 2024-11-28T01:41:22,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758142552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758142554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741966_1142 (size=30955) 2024-11-28T01:41:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758142674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758142674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758142733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758142733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758142743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:41:22,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758142882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758142882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-28T01:41:22,933 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:22,933 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-28T01:41:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T01:41:22,937 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:22,938 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:22,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:22,982 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ddf25d33241f45cfaf8ae98f3dd4952f 2024-11-28T01:41:23,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/68ee0f198bc44e1aaa9dfaedd8a333da is 50, key is test_row_0/B:col10/1732758082450/Put/seqid=0 2024-11-28T01:41:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T01:41:23,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741967_1143 (size=12001) 2024-11-28T01:41:23,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/68ee0f198bc44e1aaa9dfaedd8a333da 2024-11-28T01:41:23,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/152e0870047040ef90897405ddb8dafc is 50, key is test_row_0/C:col10/1732758082450/Put/seqid=0 2024-11-28T01:41:23,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:23,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:23,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741968_1144 (size=12001) 2024-11-28T01:41:23,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/152e0870047040ef90897405ddb8dafc 2024-11-28T01:41:23,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ddf25d33241f45cfaf8ae98f3dd4952f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ddf25d33241f45cfaf8ae98f3dd4952f 2024-11-28T01:41:23,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ddf25d33241f45cfaf8ae98f3dd4952f, entries=150, sequenceid=63, filesize=30.2 K 2024-11-28T01:41:23,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/68ee0f198bc44e1aaa9dfaedd8a333da as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/68ee0f198bc44e1aaa9dfaedd8a333da 2024-11-28T01:41:23,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/68ee0f198bc44e1aaa9dfaedd8a333da, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T01:41:23,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/152e0870047040ef90897405ddb8dafc as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/152e0870047040ef90897405ddb8dafc 2024-11-28T01:41:23,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/152e0870047040ef90897405ddb8dafc, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T01:41:23,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 95ac6ec43a1457c3aa9b4726f69c1394 in 722ms, sequenceid=63, compaction requested=true 2024-11-28T01:41:23,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:23,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:23,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:23,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:23,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:23,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:23,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:41:23,178 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:23,178 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:23,179 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:23,179 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:23,179 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,179 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/0952565ebb354cd2bb3f9078dc3c6b1e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/593138f401bf42e9aa33d95d0c36604c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/68ee0f198bc44e1aaa9dfaedd8a333da] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=35.2 K 2024-11-28T01:41:23,180 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:23,180 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:23,180 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,180 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/bc8dea9c0b5e44df97119cba59e54cfa, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/f51463a53ff8427c8478b695aebc3633, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ddf25d33241f45cfaf8ae98f3dd4952f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=90.7 K 2024-11-28T01:41:23,180 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0952565ebb354cd2bb3f9078dc3c6b1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758080767 2024-11-28T01:41:23,180 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,180 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/bc8dea9c0b5e44df97119cba59e54cfa, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/f51463a53ff8427c8478b695aebc3633, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ddf25d33241f45cfaf8ae98f3dd4952f] 2024-11-28T01:41:23,181 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 593138f401bf42e9aa33d95d0c36604c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732758080884 2024-11-28T01:41:23,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8dea9c0b5e44df97119cba59e54cfa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758080767 2024-11-28T01:41:23,181 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ee0f198bc44e1aaa9dfaedd8a333da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758082094 2024-11-28T01:41:23,182 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f51463a53ff8427c8478b695aebc3633, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732758080884 2024-11-28T01:41:23,183 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf25d33241f45cfaf8ae98f3dd4952f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758082094 2024-11-28T01:41:23,199 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:23,202 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c35217c61e09437dbdb9733b85a38576 is 50, key is test_row_0/B:col10/1732758082450/Put/seqid=0 2024-11-28T01:41:23,213 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:23,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:41:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:23,228 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411280bc4f4f23923463f9b0668544f21b851_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:23,236 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411280bc4f4f23923463f9b0668544f21b851_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:23,236 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280bc4f4f23923463f9b0668544f21b851_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:23,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:23,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T01:41:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741969_1145 (size=12104) 2024-11-28T01:41:23,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758143273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741970_1146 (size=4469) 2024-11-28T01:41:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758143273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758143273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,291 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#127 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:23,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287067c524a3fa4707a59a5ed8b26a2d96_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758082552/Put/seqid=0 2024-11-28T01:41:23,293 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/4d6deea8b8914363906da6a808372ddf is 175, key is test_row_0/A:col10/1732758082450/Put/seqid=0 2024-11-28T01:41:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758143277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758143277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741971_1147 (size=14594) 2024-11-28T01:41:23,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741972_1148 (size=31058) 2024-11-28T01:41:23,384 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/4d6deea8b8914363906da6a808372ddf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/4d6deea8b8914363906da6a808372ddf 2024-11-28T01:41:23,392 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into 4d6deea8b8914363906da6a808372ddf(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:23,392 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:23,392 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=13, startTime=1732758083177; duration=0sec 2024-11-28T01:41:23,392 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:23,392 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:23,393 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:23,397 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:23,398 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:23,398 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,398 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d50333073f514ee0ac85522f616b4340, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d0e19489ea1f4431b0106c565708ca97, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/152e0870047040ef90897405ddb8dafc] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=35.2 K 2024-11-28T01:41:23,398 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d50333073f514ee0ac85522f616b4340, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758080767 2024-11-28T01:41:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,399 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e19489ea1f4431b0106c565708ca97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732758080884 2024-11-28T01:41:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758143394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,399 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152e0870047040ef90897405ddb8dafc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758082094 2024-11-28T01:41:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758143394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758143394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:23,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:23,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758143403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758143404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,422 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:23,423 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/8cd7f3e405b445f486b1fa1cf27aa089 is 50, key is test_row_0/C:col10/1732758082450/Put/seqid=0 2024-11-28T01:41:23,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741973_1149 (size=12104) 2024-11-28T01:41:23,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:23,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T01:41:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758143608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758143613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758143615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758143622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758143623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,692 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c35217c61e09437dbdb9733b85a38576 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c35217c61e09437dbdb9733b85a38576 2024-11-28T01:41:23,702 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into c35217c61e09437dbdb9733b85a38576(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:23,702 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:23,702 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=13, startTime=1732758083177; duration=0sec 2024-11-28T01:41:23,702 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:23,702 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:23,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:23,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287067c524a3fa4707a59a5ed8b26a2d96_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287067c524a3fa4707a59a5ed8b26a2d96_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:23,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b6adebfad81e4777aa8772df0b1fc3e9, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:23,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b6adebfad81e4777aa8772df0b1fc3e9 is 175, key is test_row_0/A:col10/1732758082552/Put/seqid=0 2024-11-28T01:41:23,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741974_1150 (size=39549) 2024-11-28T01:41:23,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b6adebfad81e4777aa8772df0b1fc3e9 2024-11-28T01:41:23,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/feef5acb861147748423ae41e0dbdccf is 50, key is test_row_0/B:col10/1732758082552/Put/seqid=0 2024-11-28T01:41:23,868 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/8cd7f3e405b445f486b1fa1cf27aa089 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8cd7f3e405b445f486b1fa1cf27aa089 2024-11-28T01:41:23,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:23,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:23,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:23,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:23,876 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 8cd7f3e405b445f486b1fa1cf27aa089(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:23,876 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:23,876 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=13, startTime=1732758083177; duration=0sec 2024-11-28T01:41:23,876 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:23,876 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:23,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741975_1151 (size=12001) 2024-11-28T01:41:23,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/feef5acb861147748423ae41e0dbdccf 2024-11-28T01:41:23,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/9d1ae3bc78ec4d20a09a30a5b303d33c is 50, key is test_row_0/C:col10/1732758082552/Put/seqid=0 2024-11-28T01:41:23,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758143923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758143923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758143932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758143943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758143943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741976_1152 (size=12001) 2024-11-28T01:41:23,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/9d1ae3bc78ec4d20a09a30a5b303d33c 2024-11-28T01:41:23,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b6adebfad81e4777aa8772df0b1fc3e9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b6adebfad81e4777aa8772df0b1fc3e9 2024-11-28T01:41:23,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b6adebfad81e4777aa8772df0b1fc3e9, entries=200, sequenceid=83, filesize=38.6 K 2024-11-28T01:41:23,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/feef5acb861147748423ae41e0dbdccf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/feef5acb861147748423ae41e0dbdccf 2024-11-28T01:41:24,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/feef5acb861147748423ae41e0dbdccf, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T01:41:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/9d1ae3bc78ec4d20a09a30a5b303d33c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/9d1ae3bc78ec4d20a09a30a5b303d33c 2024-11-28T01:41:24,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/9d1ae3bc78ec4d20a09a30a5b303d33c, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T01:41:24,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:24,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:24,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:24,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:24,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 95ac6ec43a1457c3aa9b4726f69c1394 in 809ms, sequenceid=83, compaction requested=false 2024-11-28T01:41:24,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:24,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:24,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T01:41:24,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T01:41:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:24,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:41:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:24,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b126d38fb2cd4f0b9cdd1c0178a98249_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758083270/Put/seqid=0 2024-11-28T01:41:24,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741977_1153 (size=12154) 2024-11-28T01:41:24,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,280 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b126d38fb2cd4f0b9cdd1c0178a98249_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b126d38fb2cd4f0b9cdd1c0178a98249_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:24,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ef84ee79371946f881eb2ddb63e9553b, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:24,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ef84ee79371946f881eb2ddb63e9553b is 175, key is test_row_0/A:col10/1732758083270/Put/seqid=0 2024-11-28T01:41:24,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741978_1154 (size=30955) 2024-11-28T01:41:24,306 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ef84ee79371946f881eb2ddb63e9553b 2024-11-28T01:41:24,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c08b70a4965342f0a5ffc38ff4570c7f is 50, key is test_row_0/B:col10/1732758083270/Put/seqid=0 2024-11-28T01:41:24,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741979_1155 (size=12001) 2024-11-28T01:41:24,380 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c08b70a4965342f0a5ffc38ff4570c7f 2024-11-28T01:41:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/067292ca94cb435c8057f230630dcd1c is 50, key is test_row_0/C:col10/1732758083270/Put/seqid=0 2024-11-28T01:41:24,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741980_1156 (size=12001) 2024-11-28T01:41:24,439 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/067292ca94cb435c8057f230630dcd1c 2024-11-28T01:41:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ef84ee79371946f881eb2ddb63e9553b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ef84ee79371946f881eb2ddb63e9553b 2024-11-28T01:41:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:24,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:24,458 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ef84ee79371946f881eb2ddb63e9553b, entries=150, sequenceid=99, filesize=30.2 K 2024-11-28T01:41:24,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c08b70a4965342f0a5ffc38ff4570c7f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c08b70a4965342f0a5ffc38ff4570c7f 2024-11-28T01:41:24,468 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c08b70a4965342f0a5ffc38ff4570c7f, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T01:41:24,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/067292ca94cb435c8057f230630dcd1c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/067292ca94cb435c8057f230630dcd1c 2024-11-28T01:41:24,477 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/067292ca94cb435c8057f230630dcd1c, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T01:41:24,478 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=40.25 KB/41220 for 95ac6ec43a1457c3aa9b4726f69c1394 in 301ms, sequenceid=99, compaction requested=true 2024-11-28T01:41:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-28T01:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-28T01:41:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:24,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T01:41:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:24,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:24,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-28T01:41:24,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-28T01:41:24,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.5520 sec 2024-11-28T01:41:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128950be47f28e342ca8595addc7346fb8c_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758084467/Put/seqid=0 2024-11-28T01:41:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741981_1157 (size=21918) 2024-11-28T01:41:24,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,563 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128950be47f28e342ca8595addc7346fb8c_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128950be47f28e342ca8595addc7346fb8c_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,565 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/612864eebb8c4ae4b81d5dc11c3a79c4, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:24,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/612864eebb8c4ae4b81d5dc11c3a79c4 is 175, key is test_row_0/A:col10/1732758084467/Put/seqid=0 2024-11-28T01:41:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:24,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732758084498, max timestamp=1732758084597Num uniques 859; 2024-11-28T01:41:24,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-28T01:41:24,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758144569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732758084498, max timestamp=1732758084597Num uniques 859; 2024-11-28T01:41:24,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-28T01:41:24,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1732758084498, max timestamp=1732758084597Num uniques 859; 2024-11-28T01:41:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-28T01:41:24,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741982_1158 (size=65323) 2024-11-28T01:41:24,606 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/612864eebb8c4ae4b81d5dc11c3a79c4 2024-11-28T01:41:24,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/114ee791f27549f28da3fea3580f525f is 50, key is test_row_0/B:col10/1732758084467/Put/seqid=0 2024-11-28T01:41:24,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758144613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758144617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758144623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758144626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741983_1159 (size=21365) 2024-11-28T01:41:24,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/114ee791f27549f28da3fea3580f525f 2024-11-28T01:41:24,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/76f682106b8d482ab7d78eb4e16f52a1 is 50, key is test_row_0/C:col10/1732758084467/Put/seqid=0 2024-11-28T01:41:24,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758144713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758144733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758144733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758144742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741984_1160 (size=12001) 2024-11-28T01:41:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758144753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758144933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758144943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758144943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758144963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758144972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T01:41:25,083 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:25,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-28T01:41:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T01:41:25,087 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:25,089 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:25,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:25,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/76f682106b8d482ab7d78eb4e16f52a1 2024-11-28T01:41:25,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/612864eebb8c4ae4b81d5dc11c3a79c4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/612864eebb8c4ae4b81d5dc11c3a79c4 2024-11-28T01:41:25,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/612864eebb8c4ae4b81d5dc11c3a79c4, entries=350, sequenceid=110, filesize=63.8 K 2024-11-28T01:41:25,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/114ee791f27549f28da3fea3580f525f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/114ee791f27549f28da3fea3580f525f 2024-11-28T01:41:25,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/114ee791f27549f28da3fea3580f525f, entries=350, sequenceid=110, filesize=20.9 K 2024-11-28T01:41:25,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/76f682106b8d482ab7d78eb4e16f52a1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/76f682106b8d482ab7d78eb4e16f52a1 2024-11-28T01:41:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T01:41:25,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/76f682106b8d482ab7d78eb4e16f52a1, entries=150, sequenceid=110, filesize=11.7 K 2024-11-28T01:41:25,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=154.31 KB/158010 for 95ac6ec43a1457c3aa9b4726f69c1394 in 724ms, sequenceid=110, compaction requested=true 2024-11-28T01:41:25,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:25,206 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:25,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:25,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:25,207 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:25,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:25,208 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:25,208 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:25,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:25,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:25,208 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:25,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:25,208 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/4d6deea8b8914363906da6a808372ddf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b6adebfad81e4777aa8772df0b1fc3e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ef84ee79371946f881eb2ddb63e9553b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/612864eebb8c4ae4b81d5dc11c3a79c4] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=163.0 K 2024-11-28T01:41:25,208 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:25,208 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/4d6deea8b8914363906da6a808372ddf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b6adebfad81e4777aa8772df0b1fc3e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ef84ee79371946f881eb2ddb63e9553b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/612864eebb8c4ae4b81d5dc11c3a79c4] 2024-11-28T01:41:25,209 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57471 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:25,209 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:25,209 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:25,209 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c35217c61e09437dbdb9733b85a38576, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/feef5acb861147748423ae41e0dbdccf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c08b70a4965342f0a5ffc38ff4570c7f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/114ee791f27549f28da3fea3580f525f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=56.1 K 2024-11-28T01:41:25,209 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d6deea8b8914363906da6a808372ddf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758082094 2024-11-28T01:41:25,210 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c35217c61e09437dbdb9733b85a38576, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758082094 2024-11-28T01:41:25,210 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6adebfad81e4777aa8772df0b1fc3e9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758082547 2024-11-28T01:41:25,210 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting feef5acb861147748423ae41e0dbdccf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758082547 2024-11-28T01:41:25,210 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef84ee79371946f881eb2ddb63e9553b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758083270 2024-11-28T01:41:25,211 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 612864eebb8c4ae4b81d5dc11c3a79c4, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732758084447 2024-11-28T01:41:25,211 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c08b70a4965342f0a5ffc38ff4570c7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758083270 2024-11-28T01:41:25,211 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 114ee791f27549f28da3fea3580f525f, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732758084447 2024-11-28T01:41:25,221 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:25,231 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:25,232 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/9f32769a4b4a4f4d8e3b077b8aea5d13 is 50, key is test_row_0/B:col10/1732758084467/Put/seqid=0 2024-11-28T01:41:25,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T01:41:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:25,245 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-28T01:41:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:25,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:25,246 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112894ad80e4130445b482065ebabdf69ea8_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:25,251 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112894ad80e4130445b482065ebabdf69ea8_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:25,251 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112894ad80e4130445b482065ebabdf69ea8_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:25,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758145271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128509fe14096394695bba4555052845ad3_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758084617/Put/seqid=0 2024-11-28T01:41:25,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758145272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741985_1161 (size=12241) 2024-11-28T01:41:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758145275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741986_1162 (size=4469) 2024-11-28T01:41:25,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758145276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758145283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,293 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#138 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,294 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d3c80b68bcc8429bb1944b5da58c092c is 175, key is test_row_0/A:col10/1732758084467/Put/seqid=0 2024-11-28T01:41:25,295 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/9f32769a4b4a4f4d8e3b077b8aea5d13 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9f32769a4b4a4f4d8e3b077b8aea5d13 2024-11-28T01:41:25,304 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into 9f32769a4b4a4f4d8e3b077b8aea5d13(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:25,304 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:25,304 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=12, startTime=1732758085207; duration=0sec 2024-11-28T01:41:25,305 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:25,305 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:25,305 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:25,308 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:25,308 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:25,308 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:25,308 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8cd7f3e405b445f486b1fa1cf27aa089, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/9d1ae3bc78ec4d20a09a30a5b303d33c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/067292ca94cb435c8057f230630dcd1c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/76f682106b8d482ab7d78eb4e16f52a1] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=47.0 K 2024-11-28T01:41:25,308 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cd7f3e405b445f486b1fa1cf27aa089, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758082094 2024-11-28T01:41:25,309 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1ae3bc78ec4d20a09a30a5b303d33c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758082547 2024-11-28T01:41:25,309 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 067292ca94cb435c8057f230630dcd1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758083270 2024-11-28T01:41:25,310 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f682106b8d482ab7d78eb4e16f52a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732758084467 2024-11-28T01:41:25,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741987_1163 (size=12254) 2024-11-28T01:41:25,347 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#141 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:25,347 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/92ddac8bff754916bd163cc1c5f8eebe is 50, key is test_row_0/C:col10/1732758084467/Put/seqid=0 2024-11-28T01:41:25,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741988_1164 (size=31195) 2024-11-28T01:41:25,371 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d3c80b68bcc8429bb1944b5da58c092c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d3c80b68bcc8429bb1944b5da58c092c 2024-11-28T01:41:25,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741989_1165 (size=12241) 2024-11-28T01:41:25,379 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/92ddac8bff754916bd163cc1c5f8eebe as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/92ddac8bff754916bd163cc1c5f8eebe 2024-11-28T01:41:25,383 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into d3c80b68bcc8429bb1944b5da58c092c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:25,385 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:25,385 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=12, startTime=1732758085206; duration=0sec 2024-11-28T01:41:25,385 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 92ddac8bff754916bd163cc1c5f8eebe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:25,386 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:25,386 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=12, startTime=1732758085208; duration=0sec 2024-11-28T01:41:25,386 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:25,386 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:25,386 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:25,386 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:25,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758145383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758145392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T01:41:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758145403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758145592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758145613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758145622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T01:41:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:25,748 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128509fe14096394695bba4555052845ad3_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128509fe14096394695bba4555052845ad3_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:25,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/73ff25fd3fe546519e3bf9518fe5029f, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:25,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/73ff25fd3fe546519e3bf9518fe5029f is 175, key is test_row_0/A:col10/1732758084617/Put/seqid=0 2024-11-28T01:41:25,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741990_1166 (size=31055) 2024-11-28T01:41:25,778 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/73ff25fd3fe546519e3bf9518fe5029f 2024-11-28T01:41:25,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758145783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/078c9b050a8f495dbfda4891b6233bc3 is 50, key is test_row_0/B:col10/1732758084617/Put/seqid=0 2024-11-28T01:41:25,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758145802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741991_1167 (size=12101) 2024-11-28T01:41:25,864 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/078c9b050a8f495dbfda4891b6233bc3 2024-11-28T01:41:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/761c0d0767414ba9b17864e8d116636d is 50, key is test_row_0/C:col10/1732758084617/Put/seqid=0 2024-11-28T01:41:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758145902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741992_1168 (size=12101) 2024-11-28T01:41:25,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758145933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:25,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758145942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T01:41:26,331 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/761c0d0767414ba9b17864e8d116636d 2024-11-28T01:41:26,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/73ff25fd3fe546519e3bf9518fe5029f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/73ff25fd3fe546519e3bf9518fe5029f 2024-11-28T01:41:26,348 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/73ff25fd3fe546519e3bf9518fe5029f, entries=150, sequenceid=136, filesize=30.3 K 2024-11-28T01:41:26,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-28T01:41:26,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/078c9b050a8f495dbfda4891b6233bc3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/078c9b050a8f495dbfda4891b6233bc3 2024-11-28T01:41:26,356 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/078c9b050a8f495dbfda4891b6233bc3, entries=150, sequenceid=136, filesize=11.8 K 2024-11-28T01:41:26,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/761c0d0767414ba9b17864e8d116636d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/761c0d0767414ba9b17864e8d116636d 2024-11-28T01:41:26,365 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/761c0d0767414ba9b17864e8d116636d, entries=150, sequenceid=136, filesize=11.8 K 2024-11-28T01:41:26,370 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=53.67 KB/54960 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1126ms, sequenceid=136, compaction requested=false 2024-11-28T01:41:26,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:26,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:26,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-28T01:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-28T01:41:26,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-28T01:41:26,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2860 sec 2024-11-28T01:41:26,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2970 sec 2024-11-28T01:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:26,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:41:26,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:26,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:26,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:26,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:26,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:26,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:26,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112872fb1d0c8b7c487c837635cce74c0c66_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758085271/Put/seqid=0 2024-11-28T01:41:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758086474, max timestamp=1732758086505Num uniques 859; 2024-11-28T01:41:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758086474, max timestamp=1732758086505Num uniques 859; 2024-11-28T01:41:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:26,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:26,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758086493, max timestamp=1732758086505Num uniques 859; 2024-11-28T01:41:26,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:26,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741993_1169 (size=14794) 2024-11-28T01:41:26,521 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:26,540 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112872fb1d0c8b7c487c837635cce74c0c66_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112872fb1d0c8b7c487c837635cce74c0c66_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:26,543 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/c1d470013edd4503838e8f698afa4901, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:26,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/c1d470013edd4503838e8f698afa4901 is 175, key is test_row_0/A:col10/1732758085271/Put/seqid=0 2024-11-28T01:41:26,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758146551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758146551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741994_1170 (size=39749) 2024-11-28T01:41:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758146553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,563 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/c1d470013edd4503838e8f698afa4901 2024-11-28T01:41:26,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/77e52b8055f84016bfc2a9d4909ef31b is 50, key is test_row_0/B:col10/1732758085271/Put/seqid=0 2024-11-28T01:41:26,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741995_1171 (size=12151) 2024-11-28T01:41:26,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/77e52b8055f84016bfc2a9d4909ef31b 2024-11-28T01:41:26,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/3ec64f176162413f9572a8bd0ce443a1 is 50, key is test_row_0/C:col10/1732758085271/Put/seqid=0 2024-11-28T01:41:26,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758146673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758146673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758146673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741996_1172 (size=12151) 2024-11-28T01:41:26,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/3ec64f176162413f9572a8bd0ce443a1 2024-11-28T01:41:26,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/c1d470013edd4503838e8f698afa4901 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c1d470013edd4503838e8f698afa4901 2024-11-28T01:41:26,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c1d470013edd4503838e8f698afa4901, entries=200, sequenceid=151, filesize=38.8 K 2024-11-28T01:41:26,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/77e52b8055f84016bfc2a9d4909ef31b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/77e52b8055f84016bfc2a9d4909ef31b 2024-11-28T01:41:26,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/77e52b8055f84016bfc2a9d4909ef31b, entries=150, sequenceid=151, filesize=11.9 K 2024-11-28T01:41:26,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/3ec64f176162413f9572a8bd0ce443a1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3ec64f176162413f9572a8bd0ce443a1 2024-11-28T01:41:26,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3ec64f176162413f9572a8bd0ce443a1, entries=150, sequenceid=151, filesize=11.9 K 2024-11-28T01:41:26,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for 95ac6ec43a1457c3aa9b4726f69c1394 in 282ms, sequenceid=151, compaction requested=true 2024-11-28T01:41:26,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:26,716 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:26,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:26,717 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:26,717 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:26,717 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:26,718 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d3c80b68bcc8429bb1944b5da58c092c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/73ff25fd3fe546519e3bf9518fe5029f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c1d470013edd4503838e8f698afa4901] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=99.6 K 2024-11-28T01:41:26,718 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:26,718 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d3c80b68bcc8429bb1944b5da58c092c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/73ff25fd3fe546519e3bf9518fe5029f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c1d470013edd4503838e8f698afa4901] 2024-11-28T01:41:26,718 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3c80b68bcc8429bb1944b5da58c092c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732758084467 2024-11-28T01:41:26,719 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ff25fd3fe546519e3bf9518fe5029f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732758084569 2024-11-28T01:41:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:26,719 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:26,719 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d470013edd4503838e8f698afa4901, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732758085266 2024-11-28T01:41:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:26,721 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:26,721 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:26,721 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:26,721 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9f32769a4b4a4f4d8e3b077b8aea5d13, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/078c9b050a8f495dbfda4891b6233bc3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/77e52b8055f84016bfc2a9d4909ef31b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=35.6 K 2024-11-28T01:41:26,722 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f32769a4b4a4f4d8e3b077b8aea5d13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732758084467 2024-11-28T01:41:26,723 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 078c9b050a8f495dbfda4891b6233bc3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732758084569 2024-11-28T01:41:26,724 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e52b8055f84016bfc2a9d4909ef31b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732758085266 2024-11-28T01:41:26,739 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:26,744 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411285f5039988a4945c28137dc3881576ab2_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:26,745 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:26,745 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/6a824c4824974319bb0263f9c8ac63c4 is 50, key is test_row_0/B:col10/1732758085271/Put/seqid=0 2024-11-28T01:41:26,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411285f5039988a4945c28137dc3881576ab2_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:26,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285f5039988a4945c28137dc3881576ab2_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:26,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741997_1173 (size=12493) 2024-11-28T01:41:26,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741998_1174 (size=4469) 2024-11-28T01:41:26,773 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#147 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:26,774 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/30a80d4f1e5648af9a17210ca8207b96 is 175, key is test_row_0/A:col10/1732758085271/Put/seqid=0 2024-11-28T01:41:26,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741999_1175 (size=31447) 2024-11-28T01:41:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:26,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-28T01:41:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:26,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:26,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284990364eaf9d4eba83dbd688a8011c05_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758086551/Put/seqid=0 2024-11-28T01:41:26,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742000_1176 (size=14794) 2024-11-28T01:41:26,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:26,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758146850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758146861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284990364eaf9d4eba83dbd688a8011c05_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284990364eaf9d4eba83dbd688a8011c05_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:26,864 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/6d4930b033664edfab454a1989269837, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/6d4930b033664edfab454a1989269837 is 175, key is test_row_0/A:col10/1732758086551/Put/seqid=0 2024-11-28T01:41:26,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758146883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758146883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758146883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742001_1177 (size=39749) 2024-11-28T01:41:26,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/6d4930b033664edfab454a1989269837 2024-11-28T01:41:26,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/5c6f8d6162aa4c79a2021197fe78ead5 is 50, key is test_row_0/B:col10/1732758086551/Put/seqid=0 2024-11-28T01:41:26,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742002_1178 (size=12151) 2024-11-28T01:41:26,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758146963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:26,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758146973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:26,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,167 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/6a824c4824974319bb0263f9c8ac63c4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/6a824c4824974319bb0263f9c8ac63c4 2024-11-28T01:41:27,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758147173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,176 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into 6a824c4824974319bb0263f9c8ac63c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:27,176 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:27,176 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=13, startTime=1732758086719; duration=0sec 2024-11-28T01:41:27,176 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:27,176 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:27,176 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:27,179 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:27,179 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:27,179 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,180 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/92ddac8bff754916bd163cc1c5f8eebe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/761c0d0767414ba9b17864e8d116636d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3ec64f176162413f9572a8bd0ce443a1] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=35.6 K 2024-11-28T01:41:27,180 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ddac8bff754916bd163cc1c5f8eebe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732758084467 2024-11-28T01:41:27,180 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 761c0d0767414ba9b17864e8d116636d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732758084569 2024-11-28T01:41:27,181 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec64f176162413f9572a8bd0ce443a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732758085266 2024-11-28T01:41:27,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758147184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,197 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/30a80d4f1e5648af9a17210ca8207b96 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/30a80d4f1e5648af9a17210ca8207b96 2024-11-28T01:41:27,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758147192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758147192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758147193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,206 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into 30a80d4f1e5648af9a17210ca8207b96(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:27,206 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:27,206 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=13, startTime=1732758086716; duration=0sec 2024-11-28T01:41:27,207 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:27,207 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:27,213 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:27,214 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/3009a7469aa5438196f8ddc18701f5c8 is 50, key is test_row_0/C:col10/1732758085271/Put/seqid=0 2024-11-28T01:41:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T01:41:27,234 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:27,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-28T01:41:27,237 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T01:41:27,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:27,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:27,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742003_1179 (size=12493) 2024-11-28T01:41:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T01:41:27,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/5c6f8d6162aa4c79a2021197fe78ead5 2024-11-28T01:41:27,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/4ab6b16f9d3d4d26ab95d77d932091ae is 50, key is test_row_0/C:col10/1732758086551/Put/seqid=0 2024-11-28T01:41:27,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:27,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:27,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742004_1180 (size=12151) 2024-11-28T01:41:27,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/4ab6b16f9d3d4d26ab95d77d932091ae 2024-11-28T01:41:27,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/6d4930b033664edfab454a1989269837 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/6d4930b033664edfab454a1989269837 2024-11-28T01:41:27,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/6d4930b033664edfab454a1989269837, entries=200, sequenceid=186, filesize=38.8 K 2024-11-28T01:41:27,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/5c6f8d6162aa4c79a2021197fe78ead5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5c6f8d6162aa4c79a2021197fe78ead5 2024-11-28T01:41:27,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5c6f8d6162aa4c79a2021197fe78ead5, entries=150, sequenceid=186, filesize=11.9 K 2024-11-28T01:41:27,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/4ab6b16f9d3d4d26ab95d77d932091ae as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4ab6b16f9d3d4d26ab95d77d932091ae 2024-11-28T01:41:27,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4ab6b16f9d3d4d26ab95d77d932091ae, entries=150, sequenceid=186, filesize=11.9 K 2024-11-28T01:41:27,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for 95ac6ec43a1457c3aa9b4726f69c1394 in 666ms, sequenceid=186, compaction requested=false 2024-11-28T01:41:27,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:27,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:41:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:27,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112815a52f5131db4071b02df941182e676a_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758086839/Put/seqid=0 2024-11-28T01:41:27,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T01:41:27,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742005_1181 (size=14794) 2024-11-28T01:41:27,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758147577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758147578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,676 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/3009a7469aa5438196f8ddc18701f5c8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3009a7469aa5438196f8ddc18701f5c8 2024-11-28T01:41:27,685 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 3009a7469aa5438196f8ddc18701f5c8(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:27,685 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:27,685 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=13, startTime=1732758086719; duration=0sec 2024-11-28T01:41:27,686 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:27,686 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:27,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758147692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758147692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758147713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758147713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758147713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:27,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T01:41:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:27,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758147913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758147913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:27,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:27,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112815a52f5131db4071b02df941182e676a_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112815a52f5131db4071b02df941182e676a_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:27,993 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/7743831214954f04a35042d996fe778c, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:27,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/7743831214954f04a35042d996fe778c is 175, key is test_row_0/A:col10/1732758086839/Put/seqid=0 2024-11-28T01:41:28,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742006_1182 (size=39749) 2024-11-28T01:41:28,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:28,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:28,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758148224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:28,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:28,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758148232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:28,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T01:41:28,415 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/7743831214954f04a35042d996fe778c 2024-11-28T01:41:28,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/918bed97866a42ed96a32a1839d83e2c is 50, key is test_row_0/B:col10/1732758086839/Put/seqid=0 2024-11-28T01:41:28,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742007_1183 (size=12151) 2024-11-28T01:41:28,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/918bed97866a42ed96a32a1839d83e2c 2024-11-28T01:41:28,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/b884cc031b204f08abacc41bcb326c3b is 50, key is test_row_0/C:col10/1732758086839/Put/seqid=0 2024-11-28T01:41:28,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742008_1184 (size=12151) 2024-11-28T01:41:28,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:28,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:28,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758148722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:28,729 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_0' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758148733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758148733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:28,738 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:28,738 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:28,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758148743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:28,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758148743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:28,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:28,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:28,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/b884cc031b204f08abacc41bcb326c3b 2024-11-28T01:41:28,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/7743831214954f04a35042d996fe778c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/7743831214954f04a35042d996fe778c 2024-11-28T01:41:28,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:28,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/7743831214954f04a35042d996fe778c, entries=200, sequenceid=200, filesize=38.8 K 2024-11-28T01:41:28,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/918bed97866a42ed96a32a1839d83e2c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/918bed97866a42ed96a32a1839d83e2c 2024-11-28T01:41:28,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/918bed97866a42ed96a32a1839d83e2c, entries=150, sequenceid=200, filesize=11.9 K 2024-11-28T01:41:28,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/b884cc031b204f08abacc41bcb326c3b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b884cc031b204f08abacc41bcb326c3b 2024-11-28T01:41:28,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b884cc031b204f08abacc41bcb326c3b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-28T01:41:28,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1492ms, sequenceid=200, compaction requested=true 2024-11-28T01:41:28,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:28,981 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:28,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:28,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:28,982 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:28,982 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:28,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:28,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:28,983 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:28,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:28,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:28,983 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,983 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/30a80d4f1e5648af9a17210ca8207b96, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/6d4930b033664edfab454a1989269837, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/7743831214954f04a35042d996fe778c] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=108.3 K 2024-11-28T01:41:28,983 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,983 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/30a80d4f1e5648af9a17210ca8207b96, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/6d4930b033664edfab454a1989269837, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/7743831214954f04a35042d996fe778c] 2024-11-28T01:41:28,984 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:28,984 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30a80d4f1e5648af9a17210ca8207b96, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732758085266 2024-11-28T01:41:28,984 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:28,984 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:28,984 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/6a824c4824974319bb0263f9c8ac63c4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5c6f8d6162aa4c79a2021197fe78ead5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/918bed97866a42ed96a32a1839d83e2c] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=35.9 K 2024-11-28T01:41:28,984 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d4930b033664edfab454a1989269837, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732758086543 2024-11-28T01:41:28,985 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a824c4824974319bb0263f9c8ac63c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732758085266 2024-11-28T01:41:28,985 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7743831214954f04a35042d996fe778c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732758086839 2024-11-28T01:41:28,986 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c6f8d6162aa4c79a2021197fe78ead5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732758086543 2024-11-28T01:41:28,986 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 918bed97866a42ed96a32a1839d83e2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732758086839 2024-11-28T01:41:29,022 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:29,026 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:29,027 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/4e0696351f2a4183abf585b1e021b720 is 50, key is test_row_0/B:col10/1732758086839/Put/seqid=0 2024-11-28T01:41:29,037 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112853ad70195b6c4811b10a0b1f1bf1f68a_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:29,043 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112853ad70195b6c4811b10a0b1f1bf1f68a_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:29,043 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112853ad70195b6c4811b10a0b1f1bf1f68a_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:29,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742009_1185 (size=12595) 2024-11-28T01:41:29,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742010_1186 (size=4469) 2024-11-28T01:41:29,090 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#156 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:29,091 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/c11beb472d7f44789e00a4f9efc57a27 is 175, key is test_row_0/A:col10/1732758086839/Put/seqid=0 2024-11-28T01:41:29,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T01:41:29,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:29,098 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:41:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:29,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e23fac9f497444aeb6495d7e19344efc_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758087576/Put/seqid=0 2024-11-28T01:41:29,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742011_1187 (size=31549) 2024-11-28T01:41:29,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742012_1188 (size=12304) 2024-11-28T01:41:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,203 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e23fac9f497444aeb6495d7e19344efc_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e23fac9f497444aeb6495d7e19344efc_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:29,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/64fee9bca22a4dc4b12b3913177dfee8, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/64fee9bca22a4dc4b12b3913177dfee8 is 175, key is test_row_0/A:col10/1732758087576/Put/seqid=0 2024-11-28T01:41:29,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742013_1189 (size=31105) 2024-11-28T01:41:29,246 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/64fee9bca22a4dc4b12b3913177dfee8 2024-11-28T01:41:29,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/3e4e86e6b7df41e3bb552ac55ac11038 is 50, key is test_row_0/B:col10/1732758087576/Put/seqid=0 2024-11-28T01:41:29,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742014_1190 (size=12151) 2024-11-28T01:41:29,305 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/3e4e86e6b7df41e3bb552ac55ac11038 2024-11-28T01:41:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/8ddb1098269a49e3a4082843eeac663e is 50, key is test_row_0/C:col10/1732758087576/Put/seqid=0 2024-11-28T01:41:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T01:41:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742015_1191 (size=12151) 2024-11-28T01:41:29,386 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/8ddb1098269a49e3a4082843eeac663e 2024-11-28T01:41:29,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/64fee9bca22a4dc4b12b3913177dfee8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/64fee9bca22a4dc4b12b3913177dfee8 2024-11-28T01:41:29,401 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/64fee9bca22a4dc4b12b3913177dfee8, entries=150, sequenceid=221, filesize=30.4 K 2024-11-28T01:41:29,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/3e4e86e6b7df41e3bb552ac55ac11038 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/3e4e86e6b7df41e3bb552ac55ac11038 2024-11-28T01:41:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,416 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/3e4e86e6b7df41e3bb552ac55ac11038, entries=150, sequenceid=221, filesize=11.9 K 2024-11-28T01:41:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/8ddb1098269a49e3a4082843eeac663e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8ddb1098269a49e3a4082843eeac663e 2024-11-28T01:41:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,429 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8ddb1098269a49e3a4082843eeac663e, entries=150, sequenceid=221, filesize=11.9 K 2024-11-28T01:41:29,431 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 95ac6ec43a1457c3aa9b4726f69c1394 in 332ms, sequenceid=221, compaction requested=true 2024-11-28T01:41:29,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:29,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:29,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-28T01:41:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-28T01:41:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-28T01:41:29,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-28T01:41:29,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2020 sec 2024-11-28T01:41:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,481 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/4e0696351f2a4183abf585b1e021b720 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/4e0696351f2a4183abf585b1e021b720 2024-11-28T01:41:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,493 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into 4e0696351f2a4183abf585b1e021b720(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:29,493 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,493 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=13, startTime=1732758088982; duration=0sec 2024-11-28T01:41:29,493 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:29,493 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:29,493 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,500 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:29,500 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:29,500 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:29,500 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3009a7469aa5438196f8ddc18701f5c8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4ab6b16f9d3d4d26ab95d77d932091ae, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b884cc031b204f08abacc41bcb326c3b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8ddb1098269a49e3a4082843eeac663e] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=47.8 K 2024-11-28T01:41:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,501 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3009a7469aa5438196f8ddc18701f5c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732758085266 2024-11-28T01:41:29,501 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab6b16f9d3d4d26ab95d77d932091ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732758086543 2024-11-28T01:41:29,502 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b884cc031b204f08abacc41bcb326c3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732758086839 2024-11-28T01:41:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,502 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ddb1098269a49e3a4082843eeac663e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732758087557 2024-11-28T01:41:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,523 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#161 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:29,523 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/4f8df85eb3e14e8d9e6a8c474e6396b5 is 50, key is test_row_0/C:col10/1732758087576/Put/seqid=0 2024-11-28T01:41:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742016_1192 (size=12629) 2024-11-28T01:41:29,566 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/c11beb472d7f44789e00a4f9efc57a27 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c11beb472d7f44789e00a4f9efc57a27 2024-11-28T01:41:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,591 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into c11beb472d7f44789e00a4f9efc57a27(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:29,591 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:29,591 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=13, startTime=1732758088981; duration=0sec 2024-11-28T01:41:29,591 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:29,591 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:29,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288bc0788f16f44a748e6a830eddbff14c_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758089908/Put/seqid=0 2024-11-28T01:41:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,961 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/4f8df85eb3e14e8d9e6a8c474e6396b5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4f8df85eb3e14e8d9e6a8c474e6396b5 2024-11-28T01:41:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,970 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 4f8df85eb3e14e8d9e6a8c474e6396b5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:29,970 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:29,970 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=12, startTime=1732758088983; duration=0sec 2024-11-28T01:41:29,970 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:29,970 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:30,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742017_1193 (size=17284) 2024-11-28T01:41:30,015 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:30,021 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288bc0788f16f44a748e6a830eddbff14c_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288bc0788f16f44a748e6a830eddbff14c_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:30,023 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/18ec868a204c44a29bd13b9673ab3d48, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:30,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/18ec868a204c44a29bd13b9673ab3d48 is 175, key is test_row_0/A:col10/1732758089908/Put/seqid=0 2024-11-28T01:41:30,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758089989, max timestamp=1732758090043Num uniques 859; 2024-11-28T01:41:30,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:30,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758089989, max timestamp=1732758090043Num uniques 859; 2024-11-28T01:41:30,050 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:30,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758089989, max timestamp=1732758090043Num uniques 859; 2024-11-28T01:41:30,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:30,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742018_1194 (size=48385) 2024-11-28T01:41:30,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/18ec868a204c44a29bd13b9673ab3d48 2024-11-28T01:41:30,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/a7261d62b81e435aaca1e8db46df430a is 50, key is test_row_0/B:col10/1732758089908/Put/seqid=0 2024-11-28T01:41:30,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742019_1195 (size=9757) 2024-11-28T01:41:30,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758150164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758150175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758150282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758150293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758150492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758150502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/a7261d62b81e435aaca1e8db46df430a 2024-11-28T01:41:30,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/93e83a86ddad467fa4dad90d43b37c8c is 50, key is test_row_0/C:col10/1732758089908/Put/seqid=0 2024-11-28T01:41:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742020_1196 (size=9757) 2024-11-28T01:41:30,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/93e83a86ddad467fa4dad90d43b37c8c 2024-11-28T01:41:30,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/18ec868a204c44a29bd13b9673ab3d48 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/18ec868a204c44a29bd13b9673ab3d48 2024-11-28T01:41:30,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/18ec868a204c44a29bd13b9673ab3d48, entries=250, sequenceid=233, filesize=47.3 K 2024-11-28T01:41:30,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/a7261d62b81e435aaca1e8db46df430a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a7261d62b81e435aaca1e8db46df430a 2024-11-28T01:41:30,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a7261d62b81e435aaca1e8db46df430a, entries=100, sequenceid=233, filesize=9.5 K 2024-11-28T01:41:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/93e83a86ddad467fa4dad90d43b37c8c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/93e83a86ddad467fa4dad90d43b37c8c 2024-11-28T01:41:30,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/93e83a86ddad467fa4dad90d43b37c8c, entries=100, sequenceid=233, filesize=9.5 K 2024-11-28T01:41:30,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 95ac6ec43a1457c3aa9b4726f69c1394 in 724ms, sequenceid=233, compaction requested=true 2024-11-28T01:41:30,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:30,634 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:30,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:30,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:30,635 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:30,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:30,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:30,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:30,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:30,636 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:30,636 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:30,636 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:30,636 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c11beb472d7f44789e00a4f9efc57a27, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/64fee9bca22a4dc4b12b3913177dfee8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/18ec868a204c44a29bd13b9673ab3d48] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=108.4 K 2024-11-28T01:41:30,636 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:30,636 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c11beb472d7f44789e00a4f9efc57a27, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/64fee9bca22a4dc4b12b3913177dfee8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/18ec868a204c44a29bd13b9673ab3d48] 2024-11-28T01:41:30,636 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c11beb472d7f44789e00a4f9efc57a27, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732758086839 2024-11-28T01:41:30,637 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64fee9bca22a4dc4b12b3913177dfee8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732758087557 2024-11-28T01:41:30,637 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:30,637 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ec868a204c44a29bd13b9673ab3d48, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732758089802 2024-11-28T01:41:30,637 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:30,637 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:30,637 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/4e0696351f2a4183abf585b1e021b720, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/3e4e86e6b7df41e3bb552ac55ac11038, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a7261d62b81e435aaca1e8db46df430a] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=33.7 K 2024-11-28T01:41:30,638 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0696351f2a4183abf585b1e021b720, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732758086839 2024-11-28T01:41:30,640 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4e86e6b7df41e3bb552ac55ac11038, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732758087557 2024-11-28T01:41:30,641 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a7261d62b81e435aaca1e8db46df430a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732758089881 2024-11-28T01:41:30,652 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:30,658 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:30,659 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/de75be7cd16e4027b70c641ca0b8c2f2 is 50, key is test_row_0/B:col10/1732758089908/Put/seqid=0 2024-11-28T01:41:30,659 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112887913a0a344e4b3c94d4e8dde197b625_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:30,662 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112887913a0a344e4b3c94d4e8dde197b625_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:30,662 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112887913a0a344e4b3c94d4e8dde197b625_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:30,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742021_1197 (size=12697) 2024-11-28T01:41:30,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742022_1198 (size=4469) 2024-11-28T01:41:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:30,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-28T01:41:30,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:30,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:30,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:30,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:30,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:30,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:30,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112897dcd7deaf6848d7a0ebd9a10803f83e_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758090753/Put/seqid=0 2024-11-28T01:41:30,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758150765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758150767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758150768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742023_1199 (size=14994) 2024-11-28T01:41:30,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758150802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758150813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758150882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758150882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758150883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:30,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,075 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:31,076 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/74a8e2a54cb446bcbf6ed81b7551a0e9 is 175, key is test_row_0/A:col10/1732758089908/Put/seqid=0 2024-11-28T01:41:31,078 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/de75be7cd16e4027b70c641ca0b8c2f2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/de75be7cd16e4027b70c641ca0b8c2f2 2024-11-28T01:41:31,086 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into de75be7cd16e4027b70c641ca0b8c2f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:31,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:31,086 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=13, startTime=1732758090635; duration=0sec 2024-11-28T01:41:31,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:31,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:31,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T01:41:31,087 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:41:31,088 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:41:31,088 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. because compaction request was cancelled 2024-11-28T01:41:31,088 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:31,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758151093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758151093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758151094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742024_1200 (size=31758) 2024-11-28T01:41:31,135 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/74a8e2a54cb446bcbf6ed81b7551a0e9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/74a8e2a54cb446bcbf6ed81b7551a0e9 2024-11-28T01:41:31,143 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into 74a8e2a54cb446bcbf6ed81b7551a0e9(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:31,143 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:31,143 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=13, startTime=1732758090634; duration=0sec 2024-11-28T01:41:31,143 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:31,143 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:31,184 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:31,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112897dcd7deaf6848d7a0ebd9a10803f83e_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112897dcd7deaf6848d7a0ebd9a10803f83e_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:31,192 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/a4b6c8b7bf844c85a079d970f64fa495, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:31,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/a4b6c8b7bf844c85a079d970f64fa495 is 175, key is test_row_0/A:col10/1732758090753/Put/seqid=0 2024-11-28T01:41:31,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742025_1201 (size=39949) 2024-11-28T01:41:31,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/a4b6c8b7bf844c85a079d970f64fa495 2024-11-28T01:41:31,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/045ae038263f4857abfe0d4cc3cf6432 is 50, key is test_row_0/B:col10/1732758090753/Put/seqid=0 2024-11-28T01:41:31,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742026_1202 (size=12301) 2024-11-28T01:41:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758151323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758151323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T01:41:31,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,404 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758151403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758151403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,405 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-28T01:41:31,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T01:41:31,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:31,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758151412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:31,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T01:41:31,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T01:41:31,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:31,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:31,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:31,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:31,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:31,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/045ae038263f4857abfe0d4cc3cf6432 2024-11-28T01:41:31,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/23cbd3c9db6a47ab88909bdd88cff13f is 50, key is test_row_0/C:col10/1732758090753/Put/seqid=0 2024-11-28T01:41:31,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742027_1203 (size=12301) 2024-11-28T01:41:31,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/23cbd3c9db6a47ab88909bdd88cff13f 2024-11-28T01:41:31,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T01:41:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:31,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:31,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/a4b6c8b7bf844c85a079d970f64fa495 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/a4b6c8b7bf844c85a079d970f64fa495 2024-11-28T01:41:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T01:41:31,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/a4b6c8b7bf844c85a079d970f64fa495, entries=200, sequenceid=270, filesize=39.0 K 2024-11-28T01:41:31,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/045ae038263f4857abfe0d4cc3cf6432 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/045ae038263f4857abfe0d4cc3cf6432 2024-11-28T01:41:31,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/045ae038263f4857abfe0d4cc3cf6432, entries=150, sequenceid=270, filesize=12.0 K 2024-11-28T01:41:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/23cbd3c9db6a47ab88909bdd88cff13f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/23cbd3c9db6a47ab88909bdd88cff13f 2024-11-28T01:41:31,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/23cbd3c9db6a47ab88909bdd88cff13f, entries=150, sequenceid=270, filesize=12.0 K 2024-11-28T01:41:31,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 95ac6ec43a1457c3aa9b4726f69c1394 in 992ms, sequenceid=270, compaction requested=true 2024-11-28T01:41:31,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:31,748 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T01:41:31,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:31,748 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:41:31,748 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:41:31,748 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. because compaction request was cancelled 2024-11-28T01:41:31,748 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:31,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:31,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T01:41:31,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:31,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:31,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:41:31,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:41:31,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. because compaction request was cancelled 2024-11-28T01:41:31,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:31,749 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:31,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:31,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:31,750 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:31,750 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:31,750 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:31,751 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4f8df85eb3e14e8d9e6a8c474e6396b5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/93e83a86ddad467fa4dad90d43b37c8c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/23cbd3c9db6a47ab88909bdd88cff13f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=33.9 K 2024-11-28T01:41:31,751 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f8df85eb3e14e8d9e6a8c474e6396b5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732758087557 2024-11-28T01:41:31,751 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e83a86ddad467fa4dad90d43b37c8c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732758089881 2024-11-28T01:41:31,752 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23cbd3c9db6a47ab88909bdd88cff13f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732758090135 2024-11-28T01:41:31,764 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:31,765 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/ef15db0e2e4845389f557153f51ca0ab is 50, key is test_row_0/C:col10/1732758090753/Put/seqid=0 2024-11-28T01:41:31,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742028_1204 (size=12881) 2024-11-28T01:41:31,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T01:41:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:31,871 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T01:41:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:31,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:31,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285d63561b7e594d74926b5a3d697d7c94_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758090765/Put/seqid=0 2024-11-28T01:41:31,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742029_1205 (size=12454) 2024-11-28T01:41:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:31,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:31,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732758091954, max timestamp=1732758091968Num uniques 859; 2024-11-28T01:41:31,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732758091954, max timestamp=1732758091968Num uniques 859; 2024-11-28T01:41:31,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:31,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:31,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732758091954, max timestamp=1732758091968Num uniques 859; 2024-11-28T01:41:31,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:32,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732758091988, max timestamp=1732758092013Num uniques 1068; 2024-11-28T01:41:32,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-28T01:41:32,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732758091988, max timestamp=1732758092013Num uniques 1068; 2024-11-28T01:41:32,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-28T01:41:32,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732758091988, max timestamp=1732758092013Num uniques 1068; 2024-11-28T01:41:32,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-28T01:41:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T01:41:32,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758152045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758152047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758152054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758152162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758152162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758152162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,202 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/ef15db0e2e4845389f557153f51ca0ab as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ef15db0e2e4845389f557153f51ca0ab 2024-11-28T01:41:32,217 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into ef15db0e2e4845389f557153f51ca0ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:32,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:32,217 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=13, startTime=1732758091749; duration=0sec 2024-11-28T01:41:32,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:32,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:32,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:32,301 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285d63561b7e594d74926b5a3d697d7c94_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d63561b7e594d74926b5a3d697d7c94_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/13a3d836bc09440283e53d061a91bd31, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:32,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/13a3d836bc09440283e53d061a91bd31 is 175, key is test_row_0/A:col10/1732758090765/Put/seqid=0 2024-11-28T01:41:32,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742030_1206 (size=31255) 2024-11-28T01:41:32,315 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/13a3d836bc09440283e53d061a91bd31 2024-11-28T01:41:32,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d7c0ee3ac63743d1a77ddf2d086b9eac is 50, key is test_row_0/B:col10/1732758090765/Put/seqid=0 2024-11-28T01:41:32,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742031_1207 (size=12301) 2024-11-28T01:41:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758152343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758152343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,345 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,345 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_0' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758152372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758152382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758152383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T01:41:32,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758152682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758152692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758152692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:32,735 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d7c0ee3ac63743d1a77ddf2d086b9eac 2024-11-28T01:41:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/abaa83daa7484a40bb950fec4a826d4f is 50, key is test_row_0/C:col10/1732758090765/Put/seqid=0 2024-11-28T01:41:32,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742032_1208 (size=12301) 2024-11-28T01:41:32,769 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/abaa83daa7484a40bb950fec4a826d4f 2024-11-28T01:41:32,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/13a3d836bc09440283e53d061a91bd31 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/13a3d836bc09440283e53d061a91bd31 2024-11-28T01:41:32,786 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/13a3d836bc09440283e53d061a91bd31, entries=150, sequenceid=281, filesize=30.5 K 2024-11-28T01:41:32,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d7c0ee3ac63743d1a77ddf2d086b9eac as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d7c0ee3ac63743d1a77ddf2d086b9eac 2024-11-28T01:41:32,800 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d7c0ee3ac63743d1a77ddf2d086b9eac, entries=150, sequenceid=281, filesize=12.0 K 2024-11-28T01:41:32,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/abaa83daa7484a40bb950fec4a826d4f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/abaa83daa7484a40bb950fec4a826d4f 2024-11-28T01:41:32,810 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/abaa83daa7484a40bb950fec4a826d4f, entries=150, sequenceid=281, filesize=12.0 K 2024-11-28T01:41:32,810 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 95ac6ec43a1457c3aa9b4726f69c1394 in 939ms, sequenceid=281, compaction requested=true 2024-11-28T01:41:32,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:32,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:32,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-28T01:41:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-28T01:41:32,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-28T01:41:32,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-28T01:41:32,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4140 sec 2024-11-28T01:41:32,929 INFO [master/9df3f25f5867:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T01:41:32,929 INFO [master/9df3f25f5867:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T01:41:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:33,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=154.31 KB heapSize=425.16 KB 2024-11-28T01:41:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:33,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:33,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:33,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:33,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758153221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758153229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758153236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112883fdc9cfc1af4009b9609e51fc38e4a0_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758092038/Put/seqid=0 2024-11-28T01:41:33,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742033_1209 (size=14994) 2024-11-28T01:41:33,287 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:33,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112883fdc9cfc1af4009b9609e51fc38e4a0_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883fdc9cfc1af4009b9609e51fc38e4a0_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:33,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:33,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44 is 175, key is test_row_0/A:col10/1732758092038/Put/seqid=0 2024-11-28T01:41:33,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742034_1210 (size=39949) 2024-11-28T01:41:33,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758153342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758153342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758153342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T01:41:33,553 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:33,554 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-28T01:41:33,555 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T01:41:33,556 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:33,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758153552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758153553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758153553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T01:41:33,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44 2024-11-28T01:41:33,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:33,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:33,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:33,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:33,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/454d078a9b0d48c0b5a39f98dd7e2505 is 50, key is test_row_0/B:col10/1732758092038/Put/seqid=0 2024-11-28T01:41:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742035_1211 (size=12301) 2024-11-28T01:41:33,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/454d078a9b0d48c0b5a39f98dd7e2505 2024-11-28T01:41:33,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/2f4cd083626d47a2a63c5b658a62756d is 50, key is test_row_0/C:col10/1732758092038/Put/seqid=0 2024-11-28T01:41:33,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742036_1212 (size=12301) 2024-11-28T01:41:33,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/2f4cd083626d47a2a63c5b658a62756d 2024-11-28T01:41:33,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44 2024-11-28T01:41:33,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44, entries=200, sequenceid=335, filesize=39.0 K 2024-11-28T01:41:33,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/454d078a9b0d48c0b5a39f98dd7e2505 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/454d078a9b0d48c0b5a39f98dd7e2505 2024-11-28T01:41:33,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/454d078a9b0d48c0b5a39f98dd7e2505, entries=150, sequenceid=335, filesize=12.0 K 2024-11-28T01:41:33,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/2f4cd083626d47a2a63c5b658a62756d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2f4cd083626d47a2a63c5b658a62756d 2024-11-28T01:41:33,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2f4cd083626d47a2a63c5b658a62756d, entries=150, sequenceid=335, filesize=12.0 K 2024-11-28T01:41:33,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.11 KB/435312, currentSize=33.54 KB/34350 for 95ac6ec43a1457c3aa9b4726f69c1394 in 594ms, sequenceid=335, compaction requested=true 2024-11-28T01:41:33,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:33,807 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:33,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:33,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:33,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:33,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:33,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:33,807 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:33,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:33,813 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:33,813 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:33,814 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,814 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/de75be7cd16e4027b70c641ca0b8c2f2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/045ae038263f4857abfe0d4cc3cf6432, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d7c0ee3ac63743d1a77ddf2d086b9eac, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/454d078a9b0d48c0b5a39f98dd7e2505] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=48.4 K 2024-11-28T01:41:33,817 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting de75be7cd16e4027b70c641ca0b8c2f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732758087575 2024-11-28T01:41:33,818 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 045ae038263f4857abfe0d4cc3cf6432, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732758090135 2024-11-28T01:41:33,818 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c0ee3ac63743d1a77ddf2d086b9eac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732758090758 2024-11-28T01:41:33,819 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142911 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:33,819 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 454d078a9b0d48c0b5a39f98dd7e2505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732758092038 2024-11-28T01:41:33,819 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:33,819 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,819 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/74a8e2a54cb446bcbf6ed81b7551a0e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/a4b6c8b7bf844c85a079d970f64fa495, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/13a3d836bc09440283e53d061a91bd31, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=139.6 K 2024-11-28T01:41:33,819 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,819 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/74a8e2a54cb446bcbf6ed81b7551a0e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/a4b6c8b7bf844c85a079d970f64fa495, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/13a3d836bc09440283e53d061a91bd31, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44] 2024-11-28T01:41:33,820 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a8e2a54cb446bcbf6ed81b7551a0e9, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732758087575 2024-11-28T01:41:33,821 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b6c8b7bf844c85a079d970f64fa495, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732758090135 2024-11-28T01:41:33,821 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13a3d836bc09440283e53d061a91bd31, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732758090758 2024-11-28T01:41:33,822 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab93cf8fbe9a4bd7a7bd197bc9b44f44, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732758092038 2024-11-28T01:41:33,840 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:33,841 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:33,842 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d83bec67bf67487aaa3598fdf13d1286 is 50, key is test_row_0/B:col10/1732758092038/Put/seqid=0 2024-11-28T01:41:33,851 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411282f462c91602b4f63a8a66c01c5374fe4_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:33,854 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411282f462c91602b4f63a8a66c01c5374fe4_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:33,854 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f462c91602b4f63a8a66c01c5374fe4_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:33,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742037_1213 (size=12983) 2024-11-28T01:41:33,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:33,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:33,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:33,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T01:41:33,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:33,879 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d83bec67bf67487aaa3598fdf13d1286 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d83bec67bf67487aaa3598fdf13d1286 2024-11-28T01:41:33,888 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into d83bec67bf67487aaa3598fdf13d1286(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:33,888 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:33,889 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=12, startTime=1732758093807; duration=0sec 2024-11-28T01:41:33,889 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:33,889 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:33,889 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:33,890 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:33,891 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:33,891 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:33,891 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ef15db0e2e4845389f557153f51ca0ab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/abaa83daa7484a40bb950fec4a826d4f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2f4cd083626d47a2a63c5b658a62756d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=36.6 K 2024-11-28T01:41:33,891 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ef15db0e2e4845389f557153f51ca0ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732758090135 2024-11-28T01:41:33,892 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting abaa83daa7484a40bb950fec4a826d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732758090758 2024-11-28T01:41:33,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112802878fd795af486ba4650f6674f51b50_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758093866/Put/seqid=0 2024-11-28T01:41:33,894 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4cd083626d47a2a63c5b658a62756d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732758092038 2024-11-28T01:41:33,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742038_1214 (size=4469) 2024-11-28T01:41:33,901 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#178 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:33,902 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/38de82c8a93a4abdb11e0e1261fdb5e4 is 175, key is test_row_0/A:col10/1732758092038/Put/seqid=0 2024-11-28T01:41:33,921 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:33,922 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/caf16c4a9f7d4a69bf130479dfcf328f is 50, key is test_row_0/C:col10/1732758092038/Put/seqid=0 2024-11-28T01:41:33,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742039_1215 (size=17534) 2024-11-28T01:41:33,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742040_1216 (size=31937) 2024-11-28T01:41:33,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732758093897, max timestamp=1732758093923Num uniques 859; 2024-11-28T01:41:33,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T01:41:33,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732758093897, max timestamp=1732758093923Num uniques 859; 2024-11-28T01:41:33,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T01:41:33,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732758093897, max timestamp=1732758093923Num uniques 859; 2024-11-28T01:41:33,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T01:41:33,931 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:33,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112802878fd795af486ba4650f6674f51b50_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112802878fd795af486ba4650f6674f51b50_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:33,938 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/10cb0a57a0294c98a0fa74d01452e129, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:33,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/10cb0a57a0294c98a0fa74d01452e129 is 175, key is test_row_0/A:col10/1732758093866/Put/seqid=0 2024-11-28T01:41:33,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742042_1218 (size=48639) 2024-11-28T01:41:33,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/10cb0a57a0294c98a0fa74d01452e129 2024-11-28T01:41:33,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742041_1217 (size=12983) 2024-11-28T01:41:33,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/93ba26a6f6b641c69856bc0c0ac5a47b is 50, key is test_row_0/B:col10/1732758093866/Put/seqid=0 2024-11-28T01:41:33,962 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/caf16c4a9f7d4a69bf130479dfcf328f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/caf16c4a9f7d4a69bf130479dfcf328f 2024-11-28T01:41:33,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742043_1219 (size=12301) 2024-11-28T01:41:33,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/93ba26a6f6b641c69856bc0c0ac5a47b 2024-11-28T01:41:33,967 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into caf16c4a9f7d4a69bf130479dfcf328f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:33,967 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:33,967 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=13, startTime=1732758093807; duration=0sec 2024-11-28T01:41:33,968 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:33,968 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:33,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758153960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/a2d6728194464e1ebad6f8de337abb8a is 50, key is test_row_0/C:col10/1732758093866/Put/seqid=0 2024-11-28T01:41:33,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758153968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758153969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:33,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742044_1220 (size=12301) 2024-11-28T01:41:33,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/a2d6728194464e1ebad6f8de337abb8a 2024-11-28T01:41:33,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/10cb0a57a0294c98a0fa74d01452e129 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/10cb0a57a0294c98a0fa74d01452e129 2024-11-28T01:41:33,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/10cb0a57a0294c98a0fa74d01452e129, entries=250, sequenceid=345, filesize=47.5 K 2024-11-28T01:41:34,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/93ba26a6f6b641c69856bc0c0ac5a47b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/93ba26a6f6b641c69856bc0c0ac5a47b 2024-11-28T01:41:34,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/93ba26a6f6b641c69856bc0c0ac5a47b, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T01:41:34,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/a2d6728194464e1ebad6f8de337abb8a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/a2d6728194464e1ebad6f8de337abb8a 2024-11-28T01:41:34,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/a2d6728194464e1ebad6f8de337abb8a, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T01:41:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 95ac6ec43a1457c3aa9b4726f69c1394 in 162ms, sequenceid=345, compaction requested=false 2024-11-28T01:41:34,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:34,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-28T01:41:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:34,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128823ea2dd88794853aa8a77f305a19f8c_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758093957/Put/seqid=0 2024-11-28T01:41:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758154093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758154095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758154097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742045_1221 (size=14994) 2024-11-28T01:41:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,101 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:34,107 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128823ea2dd88794853aa8a77f305a19f8c_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128823ea2dd88794853aa8a77f305a19f8c_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:34,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/069ac031f12d4e56bdc941e2c6dc6670, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:34,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/069ac031f12d4e56bdc941e2c6dc6670 is 175, key is test_row_0/A:col10/1732758093957/Put/seqid=0 2024-11-28T01:41:34,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742046_1222 (size=39949) 2024-11-28T01:41:34,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/069ac031f12d4e56bdc941e2c6dc6670 2024-11-28T01:41:34,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d114773d3c1247b98db379ff2a3189f7 is 50, key is test_row_0/B:col10/1732758093957/Put/seqid=0 2024-11-28T01:41:34,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742047_1223 (size=12301) 2024-11-28T01:41:34,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d114773d3c1247b98db379ff2a3189f7 2024-11-28T01:41:34,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/32e21a792cc5447dac0a468868cffb18 is 50, key is test_row_0/C:col10/1732758093957/Put/seqid=0 2024-11-28T01:41:34,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742048_1224 (size=12301) 2024-11-28T01:41:34,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/32e21a792cc5447dac0a468868cffb18 2024-11-28T01:41:34,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/069ac031f12d4e56bdc941e2c6dc6670 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/069ac031f12d4e56bdc941e2c6dc6670 2024-11-28T01:41:34,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/069ac031f12d4e56bdc941e2c6dc6670, entries=200, sequenceid=383, filesize=39.0 K 2024-11-28T01:41:34,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/d114773d3c1247b98db379ff2a3189f7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d114773d3c1247b98db379ff2a3189f7 2024-11-28T01:41:34,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d114773d3c1247b98db379ff2a3189f7, entries=150, sequenceid=383, filesize=12.0 K 2024-11-28T01:41:34,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/32e21a792cc5447dac0a468868cffb18 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/32e21a792cc5447dac0a468868cffb18 2024-11-28T01:41:34,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T01:41:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/32e21a792cc5447dac0a468868cffb18, entries=150, sequenceid=383, filesize=12.0 K 2024-11-28T01:41:34,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 95ac6ec43a1457c3aa9b4726f69c1394 in 116ms, sequenceid=383, compaction requested=true 2024-11-28T01:41:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:34,192 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-11-28T01:41:34,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:34,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:34,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:34,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:34,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:34,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:41:34,193 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:41:34,193 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:41:34,193 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. because compaction request was cancelled 2024-11-28T01:41:34,193 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:34,193 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:34,195 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:34,195 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:34,195 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,195 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d83bec67bf67487aaa3598fdf13d1286, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/93ba26a6f6b641c69856bc0c0ac5a47b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d114773d3c1247b98db379ff2a3189f7] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=36.7 K 2024-11-28T01:41:34,196 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d83bec67bf67487aaa3598fdf13d1286, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732758092038 2024-11-28T01:41:34,196 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ba26a6f6b641c69856bc0c0ac5a47b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758093224 2024-11-28T01:41:34,197 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d114773d3c1247b98db379ff2a3189f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732758093957 2024-11-28T01:41:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:34,210 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:34,210 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/cdaf70d0b793405da58408ddc91f4878 is 50, key is test_row_0/B:col10/1732758093957/Put/seqid=0 2024-11-28T01:41:34,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:41:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:34,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282a2785041c064370a031a1f2f15d7057_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758094206/Put/seqid=0 2024-11-28T01:41:34,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742049_1225 (size=13085) 2024-11-28T01:41:34,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742050_1226 (size=14994) 2024-11-28T01:41:34,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758094246, max timestamp=1732758094266Num uniques 859; 2024-11-28T01:41:34,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758094246, max timestamp=1732758094266Num uniques 859; 2024-11-28T01:41:34,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:34,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:34,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758094246, max timestamp=1732758094266Num uniques 859; 2024-11-28T01:41:34,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:34,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758154299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758154301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758154306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,339 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/38de82c8a93a4abdb11e0e1261fdb5e4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/38de82c8a93a4abdb11e0e1261fdb5e4 2024-11-28T01:41:34,360 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into 38de82c8a93a4abdb11e0e1261fdb5e4(size=31.2 K), total size for store is 117.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:34,360 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:34,360 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=12, startTime=1732758093806; duration=0sec 2024-11-28T01:41:34,360 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:34,360 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:34,360 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:34,361 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:34,361 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:34,361 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,362 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/caf16c4a9f7d4a69bf130479dfcf328f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/a2d6728194464e1ebad6f8de337abb8a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/32e21a792cc5447dac0a468868cffb18] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=36.7 K 2024-11-28T01:41:34,363 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf16c4a9f7d4a69bf130479dfcf328f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732758092038 2024-11-28T01:41:34,364 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d6728194464e1ebad6f8de337abb8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758093224 2024-11-28T01:41:34,364 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e21a792cc5447dac0a468868cffb18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732758093957 2024-11-28T01:41:34,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758154362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,365 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758154364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,369 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_0' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:34,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,377 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:34,378 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/97be92b19680406ca567122a1637d014 is 50, key is test_row_0/C:col10/1732758093957/Put/seqid=0 2024-11-28T01:41:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742051_1227 (size=13085) 2024-11-28T01:41:34,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758154422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758154423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758154423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758154632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758154633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758154633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:34,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:34,663 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/cdaf70d0b793405da58408ddc91f4878 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/cdaf70d0b793405da58408ddc91f4878 2024-11-28T01:41:34,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282a2785041c064370a031a1f2f15d7057_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282a2785041c064370a031a1f2f15d7057_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:34,671 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/cd1c7a97963d48d3bd9deaff81d67ae4, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:34,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/cd1c7a97963d48d3bd9deaff81d67ae4 is 175, key is test_row_0/A:col10/1732758094206/Put/seqid=0 2024-11-28T01:41:34,674 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into cdaf70d0b793405da58408ddc91f4878(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:34,674 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:34,674 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=13, startTime=1732758094193; duration=0sec 2024-11-28T01:41:34,675 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:34,675 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T01:41:34,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742052_1228 (size=39949) 2024-11-28T01:41:34,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/cd1c7a97963d48d3bd9deaff81d67ae4 2024-11-28T01:41:34,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/a64688e2ef4c4c37905d7c6cc84477ab is 50, key is test_row_0/B:col10/1732758094206/Put/seqid=0 2024-11-28T01:41:34,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742053_1229 (size=12301) 2024-11-28T01:41:34,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/a64688e2ef4c4c37905d7c6cc84477ab 2024-11-28T01:41:34,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/150c4020021243b390be2ec503b0ed0a is 50, key is test_row_0/C:col10/1732758094206/Put/seqid=0 2024-11-28T01:41:34,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742054_1230 (size=12301) 2024-11-28T01:41:34,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/150c4020021243b390be2ec503b0ed0a 2024-11-28T01:41:34,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/cd1c7a97963d48d3bd9deaff81d67ae4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/cd1c7a97963d48d3bd9deaff81d67ae4 2024-11-28T01:41:34,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/cd1c7a97963d48d3bd9deaff81d67ae4, entries=200, sequenceid=395, filesize=39.0 K 2024-11-28T01:41:34,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/a64688e2ef4c4c37905d7c6cc84477ab as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a64688e2ef4c4c37905d7c6cc84477ab 2024-11-28T01:41:34,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a64688e2ef4c4c37905d7c6cc84477ab, entries=150, sequenceid=395, filesize=12.0 K 2024-11-28T01:41:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/150c4020021243b390be2ec503b0ed0a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/150c4020021243b390be2ec503b0ed0a 2024-11-28T01:41:34,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,802 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/97be92b19680406ca567122a1637d014 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/97be92b19680406ca567122a1637d014 2024-11-28T01:41:34,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/150c4020021243b390be2ec503b0ed0a, entries=150, sequenceid=395, filesize=12.0 K 2024-11-28T01:41:34,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 95ac6ec43a1457c3aa9b4726f69c1394 in 594ms, sequenceid=395, compaction requested=true 2024-11-28T01:41:34,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:34,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:34,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:34,805 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:34,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:34,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:34,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:34,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:41:34,807 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:34,808 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:34,808 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,808 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 97be92b19680406ca567122a1637d014(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:34,808 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/38de82c8a93a4abdb11e0e1261fdb5e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/10cb0a57a0294c98a0fa74d01452e129, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/069ac031f12d4e56bdc941e2c6dc6670, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/cd1c7a97963d48d3bd9deaff81d67ae4] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=156.7 K 2024-11-28T01:41:34,808 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:34,808 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=13, startTime=1732758094193; duration=0sec 2024-11-28T01:41:34,808 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,808 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/38de82c8a93a4abdb11e0e1261fdb5e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/10cb0a57a0294c98a0fa74d01452e129, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/069ac031f12d4e56bdc941e2c6dc6670, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/cd1c7a97963d48d3bd9deaff81d67ae4] 2024-11-28T01:41:34,808 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:41:34,808 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:34,808 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:34,808 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T01:41:34,809 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 38de82c8a93a4abdb11e0e1261fdb5e4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732758092038 2024-11-28T01:41:34,811 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 10cb0a57a0294c98a0fa74d01452e129, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758093214 2024-11-28T01:41:34,811 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:41:34,811 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:41:34,811 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. because compaction request was cancelled 2024-11-28T01:41:34,811 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:34,811 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T01:41:34,812 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 069ac031f12d4e56bdc941e2c6dc6670, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732758093949 2024-11-28T01:41:34,812 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1c7a97963d48d3bd9deaff81d67ae4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732758094087 2024-11-28T01:41:34,812 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:41:34,813 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:41:34,813 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. because compaction request was cancelled 2024-11-28T01:41:34,813 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:34,824 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:34,844 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411282d7340d166c34beb88a4e34157a9d477_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:34,847 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411282d7340d166c34beb88a4e34157a9d477_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:34,847 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282d7340d166c34beb88a4e34157a9d477_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:34,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742055_1231 (size=4469) 2024-11-28T01:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:34,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-28T01:41:34,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:34,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:34,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:34,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:34,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:34,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:34,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:34,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:34,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:34,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:34,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128defcad50c28844ac80ef50ca99143a85_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758094946/Put/seqid=0 2024-11-28T01:41:34,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758154972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758154974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:34,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758154979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742056_1232 (size=17534) 2024-11-28T01:41:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758155093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758155093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758155093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:35,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:35,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,286 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#191 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:35,287 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3303980d8c654ee5a1160c8c8316d7a7 is 175, key is test_row_0/A:col10/1732758094206/Put/seqid=0 2024-11-28T01:41:35,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742057_1233 (size=32073) 2024-11-28T01:41:35,312 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3303980d8c654ee5a1160c8c8316d7a7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3303980d8c654ee5a1160c8c8316d7a7 2024-11-28T01:41:35,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758155312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758155312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758155313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,318 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into 3303980d8c654ee5a1160c8c8316d7a7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:35,318 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:35,318 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=12, startTime=1732758094805; duration=0sec 2024-11-28T01:41:35,318 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:35,318 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:35,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:35,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:35,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,422 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:35,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128defcad50c28844ac80ef50ca99143a85_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128defcad50c28844ac80ef50ca99143a85_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:35,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3c1567c7afbc4a5eaf0c68b8b1d4f617, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:35,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3c1567c7afbc4a5eaf0c68b8b1d4f617 is 175, key is test_row_0/A:col10/1732758094946/Put/seqid=0 2024-11-28T01:41:35,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742058_1234 (size=48639) 2024-11-28T01:41:35,491 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3c1567c7afbc4a5eaf0c68b8b1d4f617 2024-11-28T01:41:35,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c9705a8b61c44777ba7e92179df55424 is 50, key is test_row_0/B:col10/1732758094946/Put/seqid=0 2024-11-28T01:41:35,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742059_1235 (size=12301) 2024-11-28T01:41:35,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:35,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758155623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758155623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758155623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T01:41:35,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:35,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:35,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:35,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:35,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c9705a8b61c44777ba7e92179df55424 2024-11-28T01:41:35,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/36d25ae5d9774d3d863789eae7e15d74 is 50, key is test_row_0/C:col10/1732758094946/Put/seqid=0 2024-11-28T01:41:36,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742060_1236 (size=12301) 2024-11-28T01:41:36,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758156132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:36,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758156132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:36,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758156143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:36,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:36,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:36,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/36d25ae5d9774d3d863789eae7e15d74 2024-11-28T01:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3c1567c7afbc4a5eaf0c68b8b1d4f617 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3c1567c7afbc4a5eaf0c68b8b1d4f617 2024-11-28T01:41:36,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3c1567c7afbc4a5eaf0c68b8b1d4f617, entries=250, sequenceid=433, filesize=47.5 K 2024-11-28T01:41:36,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c9705a8b61c44777ba7e92179df55424 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c9705a8b61c44777ba7e92179df55424 2024-11-28T01:41:36,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c9705a8b61c44777ba7e92179df55424, entries=150, sequenceid=433, filesize=12.0 K 2024-11-28T01:41:36,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/36d25ae5d9774d3d863789eae7e15d74 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/36d25ae5d9774d3d863789eae7e15d74 2024-11-28T01:41:36,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/36d25ae5d9774d3d863789eae7e15d74, entries=150, sequenceid=433, filesize=12.0 K 2024-11-28T01:41:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1522ms, sequenceid=433, compaction requested=true 2024-11-28T01:41:36,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:36,471 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T01:41:36,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:36,472 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:41:36,472 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:41:36,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:36,472 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. because compaction request was cancelled 2024-11-28T01:41:36,472 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:36,472 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:36,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:36,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:36,473 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:36,474 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:36,474 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,474 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/cdaf70d0b793405da58408ddc91f4878, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a64688e2ef4c4c37905d7c6cc84477ab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c9705a8b61c44777ba7e92179df55424] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=36.8 K 2024-11-28T01:41:36,474 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:36,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,476 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting cdaf70d0b793405da58408ddc91f4878, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732758093957 2024-11-28T01:41:36,476 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:36,476 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:36,476 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,476 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/97be92b19680406ca567122a1637d014, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/150c4020021243b390be2ec503b0ed0a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/36d25ae5d9774d3d863789eae7e15d74] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=36.8 K 2024-11-28T01:41:36,477 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a64688e2ef4c4c37905d7c6cc84477ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732758094206 2024-11-28T01:41:36,477 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97be92b19680406ca567122a1637d014, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732758093957 2024-11-28T01:41:36,477 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c9705a8b61c44777ba7e92179df55424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732758094304 2024-11-28T01:41:36,478 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150c4020021243b390be2ec503b0ed0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732758094206 2024-11-28T01:41:36,479 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d25ae5d9774d3d863789eae7e15d74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732758094304 2024-11-28T01:41:36,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:36,497 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#195 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:36,498 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/77afa8de783c467da125cbbfcf0ad628 is 50, key is test_row_0/C:col10/1732758094946/Put/seqid=0 2024-11-28T01:41:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T01:41:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:36,501 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:36,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,521 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#196 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:36,522 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/367ab9132fbe45d19b7b5a66c0761dcd is 50, key is test_row_0/B:col10/1732758094946/Put/seqid=0 2024-11-28T01:41:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128884c69fc599645c6a2dcfd98d8400497_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758094975/Put/seqid=0 2024-11-28T01:41:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742062_1238 (size=13187) 2024-11-28T01:41:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742061_1237 (size=13187) 2024-11-28T01:41:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,636 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/367ab9132fbe45d19b7b5a66c0761dcd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/367ab9132fbe45d19b7b5a66c0761dcd 2024-11-28T01:41:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,641 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/77afa8de783c467da125cbbfcf0ad628 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/77afa8de783c467da125cbbfcf0ad628 2024-11-28T01:41:36,642 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into 367ab9132fbe45d19b7b5a66c0761dcd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:36,642 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:36,643 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=13, startTime=1732758096472; duration=0sec 2024-11-28T01:41:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,643 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:36,643 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,649 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 77afa8de783c467da125cbbfcf0ad628(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:36,649 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:36,649 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=13, startTime=1732758096474; duration=0sec 2024-11-28T01:41:36,649 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:36,649 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742063_1239 (size=9914) 2024-11-28T01:41:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,672 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128884c69fc599645c6a2dcfd98d8400497_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128884c69fc599645c6a2dcfd98d8400497_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3643f3657acf415f84e113f6dedd7ab6, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3643f3657acf415f84e113f6dedd7ab6 is 175, key is test_row_0/A:col10/1732758094975/Put/seqid=0 2024-11-28T01:41:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742064_1240 (size=22561) 2024-11-28T01:41:36,749 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3643f3657acf415f84e113f6dedd7ab6 2024-11-28T01:41:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/b65c6c04f52a485a926f617a2426ad0a is 50, key is test_row_0/B:col10/1732758094975/Put/seqid=0 2024-11-28T01:41:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742065_1241 (size=9857) 2024-11-28T01:41:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:37,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,207 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/b65c6c04f52a485a926f617a2426ad0a 2024-11-28T01:41:37,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/da96e01a16ea4cbbb1f4fdbab55692d5 is 50, key is test_row_0/C:col10/1732758094975/Put/seqid=0 2024-11-28T01:41:37,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758097219, max timestamp=1732758097242Num uniques 859; 2024-11-28T01:41:37,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:37,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758097219, max timestamp=1732758097242Num uniques 859; 2024-11-28T01:41:37,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:37,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758097219, max timestamp=1732758097242Num uniques 859; 2024-11-28T01:41:37,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:37,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742066_1242 (size=9857) 2024-11-28T01:41:37,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758157298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758157308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758157311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758157422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758157422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758157422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758157643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758157643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758157643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,667 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/da96e01a16ea4cbbb1f4fdbab55692d5 2024-11-28T01:41:37,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/3643f3657acf415f84e113f6dedd7ab6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3643f3657acf415f84e113f6dedd7ab6 2024-11-28T01:41:37,680 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3643f3657acf415f84e113f6dedd7ab6, entries=100, sequenceid=444, filesize=22.0 K 2024-11-28T01:41:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/b65c6c04f52a485a926f617a2426ad0a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b65c6c04f52a485a926f617a2426ad0a 2024-11-28T01:41:37,686 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b65c6c04f52a485a926f617a2426ad0a, entries=100, sequenceid=444, filesize=9.6 K 2024-11-28T01:41:37,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/da96e01a16ea4cbbb1f4fdbab55692d5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/da96e01a16ea4cbbb1f4fdbab55692d5 2024-11-28T01:41:37,693 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/da96e01a16ea4cbbb1f4fdbab55692d5, entries=100, sequenceid=444, filesize=9.6 K 2024-11-28T01:41:37,695 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1194ms, sequenceid=444, compaction requested=true 2024-11-28T01:41:37,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:37,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:37,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-28T01:41:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-28T01:41:37,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-28T01:41:37,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1410 sec 2024-11-28T01:41:37,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 4.1480 sec 2024-11-28T01:41:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T01:41:37,723 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:37,723 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-28T01:41:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T01:41:37,726 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:37,727 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:37,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T01:41:37,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T01:41:37,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:37,885 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T01:41:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:37,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128651f773bdc064d50a20d883c74123c73_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758097305/Put/seqid=0 2024-11-28T01:41:37,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742067_1243 (size=12454) 2024-11-28T01:41:37,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:37,954 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128651f773bdc064d50a20d883c74123c73_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128651f773bdc064d50a20d883c74123c73_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:37,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b935379f487e4d9f8cb4fcd1c3bc04bd, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:37,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b935379f487e4d9f8cb4fcd1c3bc04bd is 175, key is test_row_0/A:col10/1732758097305/Put/seqid=0 2024-11-28T01:41:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:37,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:37,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742068_1244 (size=31255) 2024-11-28T01:41:37,977 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=482, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b935379f487e4d9f8cb4fcd1c3bc04bd 2024-11-28T01:41:37,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758157984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758157987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758157990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/220ca155fa824af9adf5efaa2402b6e3 is 50, key is test_row_0/B:col10/1732758097305/Put/seqid=0 2024-11-28T01:41:38,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742069_1245 (size=12301) 2024-11-28T01:41:38,042 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/220ca155fa824af9adf5efaa2402b6e3 2024-11-28T01:41:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T01:41:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/fc442e8914184f27867d0649c6620db7 is 50, key is test_row_0/C:col10/1732758097305/Put/seqid=0 2024-11-28T01:41:38,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742070_1246 (size=12301) 2024-11-28T01:41:38,104 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/fc442e8914184f27867d0649c6620db7 2024-11-28T01:41:38,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/b935379f487e4d9f8cb4fcd1c3bc04bd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b935379f487e4d9f8cb4fcd1c3bc04bd 2024-11-28T01:41:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758158103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,114 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b935379f487e4d9f8cb4fcd1c3bc04bd, entries=150, sequenceid=482, filesize=30.5 K 2024-11-28T01:41:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758158103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758158103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/220ca155fa824af9adf5efaa2402b6e3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/220ca155fa824af9adf5efaa2402b6e3 2024-11-28T01:41:38,120 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/220ca155fa824af9adf5efaa2402b6e3, entries=150, sequenceid=482, filesize=12.0 K 2024-11-28T01:41:38,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/fc442e8914184f27867d0649c6620db7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/fc442e8914184f27867d0649c6620db7 2024-11-28T01:41:38,130 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/fc442e8914184f27867d0649c6620db7, entries=150, sequenceid=482, filesize=12.0 K 2024-11-28T01:41:38,132 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 95ac6ec43a1457c3aa9b4726f69c1394 in 248ms, sequenceid=482, compaction requested=true 2024-11-28T01:41:38,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:38,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-28T01:41:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-28T01:41:38,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-28T01:41:38,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 407 msec 2024-11-28T01:41:38,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 414 msec 2024-11-28T01:41:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:38,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T01:41:38,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:38,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:38,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:38,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:38,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:38,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:38,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128becaa7d1cf6e45569161e00201e21a18_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758098327/Put/seqid=0 2024-11-28T01:41:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T01:41:38,352 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:38,353 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-28T01:41:38,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T01:41:38,356 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:38,357 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:38,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:38,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742071_1247 (size=14994) 2024-11-28T01:41:38,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758098361, max timestamp=1732758098377Num uniques 859; 2024-11-28T01:41:38,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:38,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758098361, max timestamp=1732758098377Num uniques 859; 2024-11-28T01:41:38,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:38,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758098361, max timestamp=1732758098377Num uniques 859; 2024-11-28T01:41:38,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:38,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54422 deadline: 1732758158413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758158418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,423 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 95ac6ec43a1457c3aa9b4726f69c1394 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:41:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758158420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758158420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758158421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T01:41:38,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:38,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:38,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758158533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758158533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758158552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758158552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:38,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T01:41:38,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758158752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758158752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,767 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:38,773 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128becaa7d1cf6e45569161e00201e21a18_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128becaa7d1cf6e45569161e00201e21a18_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,775 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ca53760a1d4440d895e3f426f3e1cc7e, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758158773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ca53760a1d4440d895e3f426f3e1cc7e is 175, key is test_row_0/A:col10/1732758098327/Put/seqid=0 2024-11-28T01:41:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758158773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:38,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742072_1248 (size=39949) 2024-11-28T01:41:38,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:38,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:38,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:38,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T01:41:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758159072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758159072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758159083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758159083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=493, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ca53760a1d4440d895e3f426f3e1cc7e 2024-11-28T01:41:39,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c57f39809df843c98db7571f414a642e is 50, key is test_row_0/B:col10/1732758098327/Put/seqid=0 2024-11-28T01:41:39,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742073_1249 (size=12301) 2024-11-28T01:41:39,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c57f39809df843c98db7571f414a642e 2024-11-28T01:41:39,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/ce729c690c0444dab5ec0ad4b86e3310 is 50, key is test_row_0/C:col10/1732758098327/Put/seqid=0 2024-11-28T01:41:39,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:39,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742074_1250 (size=12301) 2024-11-28T01:41:39,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:39,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:39,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T01:41:39,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:39,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:39,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54368 deadline: 1732758159592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54384 deadline: 1732758159593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54450 deadline: 1732758159593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54412 deadline: 1732758159593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:39,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:41:39,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/ce729c690c0444dab5ec0ad4b86e3310 2024-11-28T01:41:39,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/ca53760a1d4440d895e3f426f3e1cc7e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ca53760a1d4440d895e3f426f3e1cc7e 2024-11-28T01:41:39,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ca53760a1d4440d895e3f426f3e1cc7e, entries=200, sequenceid=493, filesize=39.0 K 2024-11-28T01:41:39,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c57f39809df843c98db7571f414a642e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c57f39809df843c98db7571f414a642e 2024-11-28T01:41:39,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c57f39809df843c98db7571f414a642e, entries=150, sequenceid=493, filesize=12.0 K 2024-11-28T01:41:39,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/ce729c690c0444dab5ec0ad4b86e3310 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ce729c690c0444dab5ec0ad4b86e3310 2024-11-28T01:41:39,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:39,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ce729c690c0444dab5ec0ad4b86e3310, entries=150, sequenceid=493, filesize=12.0 K 2024-11-28T01:41:39,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1430ms, sequenceid=493, compaction requested=true 2024-11-28T01:41:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:39,760 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T01:41:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:39,761 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:39,762 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:39,763 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:39,763 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,763 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/367ab9132fbe45d19b7b5a66c0761dcd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b65c6c04f52a485a926f617a2426ad0a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/220ca155fa824af9adf5efaa2402b6e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c57f39809df843c98db7571f414a642e] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=46.5 K 2024-11-28T01:41:39,763 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 174477 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T01:41:39,763 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:39,763 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,763 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3303980d8c654ee5a1160c8c8316d7a7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3c1567c7afbc4a5eaf0c68b8b1d4f617, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3643f3657acf415f84e113f6dedd7ab6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b935379f487e4d9f8cb4fcd1c3bc04bd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ca53760a1d4440d895e3f426f3e1cc7e] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=170.4 K 2024-11-28T01:41:39,763 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,763 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3303980d8c654ee5a1160c8c8316d7a7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3c1567c7afbc4a5eaf0c68b8b1d4f617, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3643f3657acf415f84e113f6dedd7ab6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b935379f487e4d9f8cb4fcd1c3bc04bd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ca53760a1d4440d895e3f426f3e1cc7e] 2024-11-28T01:41:39,767 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 367ab9132fbe45d19b7b5a66c0761dcd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732758094304 2024-11-28T01:41:39,767 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3303980d8c654ee5a1160c8c8316d7a7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732758094206 2024-11-28T01:41:39,767 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b65c6c04f52a485a926f617a2426ad0a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732758094971 2024-11-28T01:41:39,769 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1567c7afbc4a5eaf0c68b8b1d4f617, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732758094294 2024-11-28T01:41:39,769 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 220ca155fa824af9adf5efaa2402b6e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758097294 2024-11-28T01:41:39,771 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3643f3657acf415f84e113f6dedd7ab6, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732758094971 2024-11-28T01:41:39,771 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c57f39809df843c98db7571f414a642e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732758097984 2024-11-28T01:41:39,772 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b935379f487e4d9f8cb4fcd1c3bc04bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758097294 2024-11-28T01:41:39,772 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca53760a1d4440d895e3f426f3e1cc7e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732758097980 2024-11-28T01:41:39,794 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:39,794 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/9a857f5be04d4ed68c0ed421334c9a29 is 50, key is test_row_0/B:col10/1732758098327/Put/seqid=0 2024-11-28T01:41:39,808 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:39,818 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128fe473444d22a4914a790e3161e004294_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:39,821 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128fe473444d22a4914a790e3161e004294_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:39,821 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fe473444d22a4914a790e3161e004294_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:39,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742075_1251 (size=13323) 2024-11-28T01:41:39,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742076_1252 (size=4469) 2024-11-28T01:41:39,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T01:41:39,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:39,902 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-28T01:41:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128240ac90589ae4182a35e7972e5af2c38_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758098418/Put/seqid=0 2024-11-28T01:41:39,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742077_1253 (size=12454) 2024-11-28T01:41:39,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:39,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128240ac90589ae4182a35e7972e5af2c38_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128240ac90589ae4182a35e7972e5af2c38_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8ef75c92b80d4abca6250a4fae46f7fe, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8ef75c92b80d4abca6250a4fae46f7fe is 175, key is test_row_0/A:col10/1732758098418/Put/seqid=0 2024-11-28T01:41:39,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742078_1254 (size=31255) 2024-11-28T01:41:39,973 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=529, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8ef75c92b80d4abca6250a4fae46f7fe 2024-11-28T01:41:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/b64cce34fdd04940a4e80ef8fa7e1968 is 50, key is test_row_0/B:col10/1732758098418/Put/seqid=0 2024-11-28T01:41:39,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742079_1255 (size=12301) 2024-11-28T01:41:40,241 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/9a857f5be04d4ed68c0ed421334c9a29 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9a857f5be04d4ed68c0ed421334c9a29 2024-11-28T01:41:40,247 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#207 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:40,248 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d50c96264f804dacb4c879bdb4637331 is 175, key is test_row_0/A:col10/1732758098327/Put/seqid=0 2024-11-28T01:41:40,252 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into 9a857f5be04d4ed68c0ed421334c9a29(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:40,252 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:40,252 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=12, startTime=1732758099761; duration=0sec 2024-11-28T01:41:40,252 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:40,252 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:40,252 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:40,254 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:40,254 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:40,254 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,254 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/77afa8de783c467da125cbbfcf0ad628, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/da96e01a16ea4cbbb1f4fdbab55692d5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/fc442e8914184f27867d0649c6620db7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ce729c690c0444dab5ec0ad4b86e3310] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=46.5 K 2024-11-28T01:41:40,255 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 77afa8de783c467da125cbbfcf0ad628, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1732758094304 2024-11-28T01:41:40,255 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting da96e01a16ea4cbbb1f4fdbab55692d5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732758094971 2024-11-28T01:41:40,255 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting fc442e8914184f27867d0649c6620db7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758097294 2024-11-28T01:41:40,256 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ce729c690c0444dab5ec0ad4b86e3310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732758097984 2024-11-28T01:41:40,270 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:40,271 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/8fb5489a3dc84c519ffa01e1b07def41 is 50, key is test_row_0/C:col10/1732758098327/Put/seqid=0 2024-11-28T01:41:40,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742080_1256 (size=32243) 2024-11-28T01:41:40,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742081_1257 (size=13323) 2024-11-28T01:41:40,317 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/8fb5489a3dc84c519ffa01e1b07def41 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8fb5489a3dc84c519ffa01e1b07def41 2024-11-28T01:41:40,328 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 8fb5489a3dc84c519ffa01e1b07def41(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:40,328 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:40,328 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=12, startTime=1732758099761; duration=0sec 2024-11-28T01:41:40,329 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:40,329 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:40,394 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/b64cce34fdd04940a4e80ef8fa7e1968 2024-11-28T01:41:40,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/f273a9db36f2455dacc9bc3e211df117 is 50, key is test_row_0/C:col10/1732758098418/Put/seqid=0 2024-11-28T01:41:40,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742082_1258 (size=12301) 2024-11-28T01:41:40,440 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/f273a9db36f2455dacc9bc3e211df117 2024-11-28T01:41:40,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8ef75c92b80d4abca6250a4fae46f7fe as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8ef75c92b80d4abca6250a4fae46f7fe 2024-11-28T01:41:40,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8ef75c92b80d4abca6250a4fae46f7fe, entries=150, sequenceid=529, filesize=30.5 K 2024-11-28T01:41:40,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/b64cce34fdd04940a4e80ef8fa7e1968 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b64cce34fdd04940a4e80ef8fa7e1968 2024-11-28T01:41:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b64cce34fdd04940a4e80ef8fa7e1968, entries=150, sequenceid=529, filesize=12.0 K 2024-11-28T01:41:40,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/f273a9db36f2455dacc9bc3e211df117 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/f273a9db36f2455dacc9bc3e211df117 2024-11-28T01:41:40,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/f273a9db36f2455dacc9bc3e211df117, entries=150, sequenceid=529, filesize=12.0 K 2024-11-28T01:41:40,476 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for 95ac6ec43a1457c3aa9b4726f69c1394 in 574ms, sequenceid=529, compaction requested=false 2024-11-28T01:41:40,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:40,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-28T01:41:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-28T01:41:40,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-28T01:41:40,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-11-28T01:41:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1290 sec 2024-11-28T01:41:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T01:41:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,503 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,503 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-28T01:41:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T01:41:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,508 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:40,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T01:41:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:40,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:40,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:40,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:40,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:40,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:40,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112808fd12b90776475baf1d0e9b0d752d84_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758100647/Put/seqid=0 2024-11-28T01:41:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,706 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d50c96264f804dacb4c879bdb4637331 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d50c96264f804dacb4c879bdb4637331 2024-11-28T01:41:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,718 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into d50c96264f804dacb4c879bdb4637331(size=31.5 K), total size for store is 62.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:40,718 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:40,718 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=11, startTime=1732758099760; duration=0sec 2024-11-28T01:41:40,718 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:40,718 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742083_1259 (size=22618) 2024-11-28T01:41:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:40,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758100733, max timestamp=1732758100742Num uniques 859; 2024-11-28T01:41:40,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:40,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758100733, max timestamp=1732758100742Num uniques 859; 2024-11-28T01:41:40,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:40,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758100733, max timestamp=1732758100742Num uniques 859; 2024-11-28T01:41:40,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:40,787 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-28T01:41:40,787 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,787 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,787 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,789 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-28T01:41:40,789 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,789 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,789 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,797 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-28T01:41:40,797 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,797 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,797 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,798 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-28T01:41:40,798 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,798 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,798 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,803 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-28T01:41:40,803 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,803 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,803 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-28T01:41:40,803 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,803 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,803 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,803 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,804 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-28T01:41:40,804 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,804 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,804 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,805 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-28T01:41:40,805 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:40,805 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,805 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:40,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:40,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:40,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:40,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T01:41:40,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:40,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:40,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:41,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:41,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:41,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T01:41:41,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112808fd12b90776475baf1d0e9b0d752d84_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808fd12b90776475baf1d0e9b0d752d84_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:41,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d79ef7f1502440628a7eba35e4195f07, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:41,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d79ef7f1502440628a7eba35e4195f07 is 175, key is test_row_0/A:col10/1732758100647/Put/seqid=0 2024-11-28T01:41:41,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742084_1260 (size=66023) 2024-11-28T01:41:41,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:41,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:41,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=541, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d79ef7f1502440628a7eba35e4195f07 2024-11-28T01:41:41,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/70718f5e551645f5af79d77d0a9d502a is 50, key is test_row_0/B:col10/1732758100647/Put/seqid=0 2024-11-28T01:41:41,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742085_1261 (size=12301) 2024-11-28T01:41:41,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:41,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:41,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T01:41:41,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:41,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:41,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:41,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:41,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/70718f5e551645f5af79d77d0a9d502a 2024-11-28T01:41:41,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/b335825f406a44d19c81ead8d3bf5a51 is 50, key is test_row_0/C:col10/1732758100647/Put/seqid=0 2024-11-28T01:41:41,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742086_1262 (size=12301) 2024-11-28T01:41:42,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:42,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:42,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:42,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. as already flushing 2024-11-28T01:41:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:42,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/b335825f406a44d19c81ead8d3bf5a51 2024-11-28T01:41:42,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/d79ef7f1502440628a7eba35e4195f07 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d79ef7f1502440628a7eba35e4195f07 2024-11-28T01:41:42,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d79ef7f1502440628a7eba35e4195f07, entries=350, sequenceid=541, filesize=64.5 K 2024-11-28T01:41:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/70718f5e551645f5af79d77d0a9d502a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/70718f5e551645f5af79d77d0a9d502a 2024-11-28T01:41:42,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/70718f5e551645f5af79d77d0a9d502a, entries=150, sequenceid=541, filesize=12.0 K 2024-11-28T01:41:42,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/b335825f406a44d19c81ead8d3bf5a51 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b335825f406a44d19c81ead8d3bf5a51 2024-11-28T01:41:42,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b335825f406a44d19c81ead8d3bf5a51, entries=150, sequenceid=541, filesize=12.0 K 2024-11-28T01:41:42,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1733ms, sequenceid=541, compaction requested=true 2024-11-28T01:41:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:42,382 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 95ac6ec43a1457c3aa9b4726f69c1394:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:42,382 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:42,383 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:42,383 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:42,383 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/A is initiating minor compaction (all files) 2024-11-28T01:41:42,383 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/B is initiating minor compaction (all files) 2024-11-28T01:41:42,383 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/A in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,383 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/B in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,383 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d50c96264f804dacb4c879bdb4637331, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8ef75c92b80d4abca6250a4fae46f7fe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d79ef7f1502440628a7eba35e4195f07] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=126.5 K 2024-11-28T01:41:42,383 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9a857f5be04d4ed68c0ed421334c9a29, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b64cce34fdd04940a4e80ef8fa7e1968, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/70718f5e551645f5af79d77d0a9d502a] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=37.0 K 2024-11-28T01:41:42,383 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,383 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d50c96264f804dacb4c879bdb4637331, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8ef75c92b80d4abca6250a4fae46f7fe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d79ef7f1502440628a7eba35e4195f07] 2024-11-28T01:41:42,384 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a857f5be04d4ed68c0ed421334c9a29, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732758097984 2024-11-28T01:41:42,384 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d50c96264f804dacb4c879bdb4637331, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732758097984 2024-11-28T01:41:42,384 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b64cce34fdd04940a4e80ef8fa7e1968, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732758098416 2024-11-28T01:41:42,384 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef75c92b80d4abca6250a4fae46f7fe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732758098416 2024-11-28T01:41:42,384 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 70718f5e551645f5af79d77d0a9d502a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732758100628 2024-11-28T01:41:42,384 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79ef7f1502440628a7eba35e4195f07, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732758100622 2024-11-28T01:41:42,391 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#B#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:42,392 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c18ad68074a440589f0c8d9e9efae2d3 is 50, key is test_row_0/B:col10/1732758100647/Put/seqid=0 2024-11-28T01:41:42,393 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:42,395 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411283ce72eae523a41ccb08b6bbb3a128c35_95ac6ec43a1457c3aa9b4726f69c1394 store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:42,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742087_1263 (size=13425) 2024-11-28T01:41:42,419 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411283ce72eae523a41ccb08b6bbb3a128c35_95ac6ec43a1457c3aa9b4726f69c1394, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:42,419 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283ce72eae523a41ccb08b6bbb3a128c35_95ac6ec43a1457c3aa9b4726f69c1394 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:42,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742088_1264 (size=4469) 2024-11-28T01:41:42,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T01:41:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,508 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T01:41:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b53a05c688a046e6895c8c662400a400_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_0/A:col10/1732758100801/Put/seqid=0 2024-11-28T01:41:42,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742089_1265 (size=12454) 2024-11-28T01:41:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T01:41:42,800 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/c18ad68074a440589f0c8d9e9efae2d3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c18ad68074a440589f0c8d9e9efae2d3 2024-11-28T01:41:42,805 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/B of 95ac6ec43a1457c3aa9b4726f69c1394 into c18ad68074a440589f0c8d9e9efae2d3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:42,805 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:42,805 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/B, priority=13, startTime=1732758102382; duration=0sec 2024-11-28T01:41:42,805 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:42,805 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:B 2024-11-28T01:41:42,805 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:41:42,806 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:41:42,806 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 95ac6ec43a1457c3aa9b4726f69c1394/C is initiating minor compaction (all files) 2024-11-28T01:41:42,806 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 95ac6ec43a1457c3aa9b4726f69c1394/C in TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:42,806 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8fb5489a3dc84c519ffa01e1b07def41, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/f273a9db36f2455dacc9bc3e211df117, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b335825f406a44d19c81ead8d3bf5a51] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp, totalSize=37.0 K 2024-11-28T01:41:42,807 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb5489a3dc84c519ffa01e1b07def41, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732758097984 2024-11-28T01:41:42,807 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f273a9db36f2455dacc9bc3e211df117, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732758098416 2024-11-28T01:41:42,807 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b335825f406a44d19c81ead8d3bf5a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732758100628 2024-11-28T01:41:42,815 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#C#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:42,815 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/965d092b5e9f43d480de884f7bc337a4 is 50, key is test_row_0/C:col10/1732758100647/Put/seqid=0 2024-11-28T01:41:42,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742090_1266 (size=13425) 2024-11-28T01:41:42,824 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 95ac6ec43a1457c3aa9b4726f69c1394#A#compaction#216 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:42,825 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/52950c1e900246e7a66d246068eba0bd is 175, key is test_row_0/A:col10/1732758100647/Put/seqid=0 2024-11-28T01:41:42,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742091_1267 (size=32345) 2024-11-28T01:41:42,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:42,926 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b53a05c688a046e6895c8c662400a400_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b53a05c688a046e6895c8c662400a400_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:42,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/00a9a757e8f140678b1cafe4d1c292d4, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:42,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/00a9a757e8f140678b1cafe4d1c292d4 is 175, key is test_row_0/A:col10/1732758100801/Put/seqid=0 2024-11-28T01:41:42,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742092_1268 (size=31255) 2024-11-28T01:41:43,224 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/965d092b5e9f43d480de884f7bc337a4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/965d092b5e9f43d480de884f7bc337a4 2024-11-28T01:41:43,229 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/C of 95ac6ec43a1457c3aa9b4726f69c1394 into 965d092b5e9f43d480de884f7bc337a4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:43,229 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:43,229 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/C, priority=13, startTime=1732758102382; duration=0sec 2024-11-28T01:41:43,229 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:43,229 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:C 2024-11-28T01:41:43,234 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/52950c1e900246e7a66d246068eba0bd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/52950c1e900246e7a66d246068eba0bd 2024-11-28T01:41:43,238 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 95ac6ec43a1457c3aa9b4726f69c1394/A of 95ac6ec43a1457c3aa9b4726f69c1394 into 52950c1e900246e7a66d246068eba0bd(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:43,238 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:43,238 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., storeName=95ac6ec43a1457c3aa9b4726f69c1394/A, priority=13, startTime=1732758102382; duration=0sec 2024-11-28T01:41:43,239 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:43,239 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 95ac6ec43a1457c3aa9b4726f69c1394:A 2024-11-28T01:41:43,332 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=577, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/00a9a757e8f140678b1cafe4d1c292d4 2024-11-28T01:41:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/129fc036e2ff476ab6cf085e37cf7efc is 50, key is test_row_0/B:col10/1732758100801/Put/seqid=0 2024-11-28T01:41:43,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742093_1269 (size=12301) 2024-11-28T01:41:43,744 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/129fc036e2ff476ab6cf085e37cf7efc 2024-11-28T01:41:43,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/2b5e0e30b54d413a98f559783fda5b22 is 50, key is test_row_0/C:col10/1732758100801/Put/seqid=0 2024-11-28T01:41:43,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742094_1270 (size=12301) 2024-11-28T01:41:43,755 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/2b5e0e30b54d413a98f559783fda5b22 2024-11-28T01:41:43,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/00a9a757e8f140678b1cafe4d1c292d4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/00a9a757e8f140678b1cafe4d1c292d4 2024-11-28T01:41:43,762 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/00a9a757e8f140678b1cafe4d1c292d4, entries=150, sequenceid=577, filesize=30.5 K 2024-11-28T01:41:43,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/129fc036e2ff476ab6cf085e37cf7efc as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/129fc036e2ff476ab6cf085e37cf7efc 2024-11-28T01:41:43,766 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/129fc036e2ff476ab6cf085e37cf7efc, entries=150, sequenceid=577, filesize=12.0 K 2024-11-28T01:41:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/2b5e0e30b54d413a98f559783fda5b22 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2b5e0e30b54d413a98f559783fda5b22 2024-11-28T01:41:43,770 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2b5e0e30b54d413a98f559783fda5b22, entries=150, sequenceid=577, filesize=12.0 K 2024-11-28T01:41:43,771 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1263ms, sequenceid=577, compaction requested=false 2024-11-28T01:41:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 95ac6ec43a1457c3aa9b4726f69c1394: 2024-11-28T01:41:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-28T01:41:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-28T01:41:43,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-28T01:41:43,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2640 sec 2024-11-28T01:41:43,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 3.2710 sec 2024-11-28T01:41:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T01:41:44,672 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:48,019 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T01:41:48,444 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-28T01:41:48,444 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:41:48,444 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:48,444 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:48,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1799 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1846 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 833 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2499 rows 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 831 2024-11-28T01:41:48,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2493 rows 2024-11-28T01:41:48,445 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:41:48,445 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:41:48,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:48,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:48,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:41:48,446 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T01:41:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T01:41:48,449 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758108449"}]},"ts":"1732758108449"} 2024-11-28T01:41:48,451 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T01:41:48,451 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T01:41:48,451 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:41:48,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, UNASSIGN}] 2024-11-28T01:41:48,453 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, UNASSIGN 2024-11-28T01:41:48,454 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:48,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, UNASSIGN because future has completed 2024-11-28T01:41:48,456 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:41:48,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:41:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T01:41:48,609 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:48,609 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:41:48,609 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 95ac6ec43a1457c3aa9b4726f69c1394, disabling compactions & flushes 2024-11-28T01:41:48,609 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:48,609 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:48,609 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. after waiting 0 ms 2024-11-28T01:41:48,609 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:48,609 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 95ac6ec43a1457c3aa9b4726f69c1394 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-28T01:41:48,609 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=A 2024-11-28T01:41:48,610 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:48,610 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=B 2024-11-28T01:41:48,610 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:48,610 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 95ac6ec43a1457c3aa9b4726f69c1394, store=C 2024-11-28T01:41:48,610 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:48,615 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112865cfdc068f6742298425d2f4459e74e2_95ac6ec43a1457c3aa9b4726f69c1394 is 50, key is test_row_1/A:col10/1732758108443/Put/seqid=0 2024-11-28T01:41:48,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742095_1271 (size=7374) 2024-11-28T01:41:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T01:41:49,019 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:49,024 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112865cfdc068f6742298425d2f4459e74e2_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112865cfdc068f6742298425d2f4459e74e2_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:49,025 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8de053f80bb743ec921cd2559a32b4e2, store: [table=TestAcidGuarantees family=A region=95ac6ec43a1457c3aa9b4726f69c1394] 2024-11-28T01:41:49,026 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8de053f80bb743ec921cd2559a32b4e2 is 175, key is test_row_1/A:col10/1732758108443/Put/seqid=0 2024-11-28T01:41:49,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742096_1272 (size=13865) 2024-11-28T01:41:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T01:41:49,432 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=584, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8de053f80bb743ec921cd2559a32b4e2 2024-11-28T01:41:49,439 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/5967054a30d04141b71cbf5d8845e261 is 50, key is test_row_1/B:col10/1732758108443/Put/seqid=0 2024-11-28T01:41:49,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742097_1273 (size=7415) 2024-11-28T01:41:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T01:41:49,843 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/5967054a30d04141b71cbf5d8845e261 2024-11-28T01:41:49,851 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/cbc172b474134feab7feb0067221ce8b is 50, key is test_row_1/C:col10/1732758108443/Put/seqid=0 2024-11-28T01:41:49,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742098_1274 (size=7415) 2024-11-28T01:41:50,255 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/cbc172b474134feab7feb0067221ce8b 2024-11-28T01:41:50,260 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/A/8de053f80bb743ec921cd2559a32b4e2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8de053f80bb743ec921cd2559a32b4e2 2024-11-28T01:41:50,263 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8de053f80bb743ec921cd2559a32b4e2, entries=50, sequenceid=584, filesize=13.5 K 2024-11-28T01:41:50,264 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/B/5967054a30d04141b71cbf5d8845e261 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5967054a30d04141b71cbf5d8845e261 2024-11-28T01:41:50,267 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5967054a30d04141b71cbf5d8845e261, entries=50, sequenceid=584, filesize=7.2 K 2024-11-28T01:41:50,268 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/.tmp/C/cbc172b474134feab7feb0067221ce8b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/cbc172b474134feab7feb0067221ce8b 2024-11-28T01:41:50,271 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/cbc172b474134feab7feb0067221ce8b, entries=50, sequenceid=584, filesize=7.2 K 2024-11-28T01:41:50,272 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1663ms, sequenceid=584, compaction requested=true 2024-11-28T01:41:50,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/bc8dea9c0b5e44df97119cba59e54cfa, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/f51463a53ff8427c8478b695aebc3633, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/4d6deea8b8914363906da6a808372ddf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ddf25d33241f45cfaf8ae98f3dd4952f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b6adebfad81e4777aa8772df0b1fc3e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ef84ee79371946f881eb2ddb63e9553b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/612864eebb8c4ae4b81d5dc11c3a79c4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d3c80b68bcc8429bb1944b5da58c092c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/73ff25fd3fe546519e3bf9518fe5029f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c1d470013edd4503838e8f698afa4901, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/30a80d4f1e5648af9a17210ca8207b96, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/6d4930b033664edfab454a1989269837, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/7743831214954f04a35042d996fe778c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c11beb472d7f44789e00a4f9efc57a27, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/64fee9bca22a4dc4b12b3913177dfee8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/18ec868a204c44a29bd13b9673ab3d48, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/74a8e2a54cb446bcbf6ed81b7551a0e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/a4b6c8b7bf844c85a079d970f64fa495, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/13a3d836bc09440283e53d061a91bd31, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/38de82c8a93a4abdb11e0e1261fdb5e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/10cb0a57a0294c98a0fa74d01452e129, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/069ac031f12d4e56bdc941e2c6dc6670, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/cd1c7a97963d48d3bd9deaff81d67ae4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3303980d8c654ee5a1160c8c8316d7a7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3c1567c7afbc4a5eaf0c68b8b1d4f617, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3643f3657acf415f84e113f6dedd7ab6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b935379f487e4d9f8cb4fcd1c3bc04bd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ca53760a1d4440d895e3f426f3e1cc7e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d50c96264f804dacb4c879bdb4637331, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8ef75c92b80d4abca6250a4fae46f7fe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d79ef7f1502440628a7eba35e4195f07] to archive 2024-11-28T01:41:50,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:41:50,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/bc8dea9c0b5e44df97119cba59e54cfa to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/bc8dea9c0b5e44df97119cba59e54cfa 2024-11-28T01:41:50,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/f51463a53ff8427c8478b695aebc3633 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/f51463a53ff8427c8478b695aebc3633 2024-11-28T01:41:50,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/4d6deea8b8914363906da6a808372ddf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/4d6deea8b8914363906da6a808372ddf 2024-11-28T01:41:50,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ddf25d33241f45cfaf8ae98f3dd4952f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ddf25d33241f45cfaf8ae98f3dd4952f 2024-11-28T01:41:50,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b6adebfad81e4777aa8772df0b1fc3e9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b6adebfad81e4777aa8772df0b1fc3e9 2024-11-28T01:41:50,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ef84ee79371946f881eb2ddb63e9553b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ef84ee79371946f881eb2ddb63e9553b 2024-11-28T01:41:50,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/612864eebb8c4ae4b81d5dc11c3a79c4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/612864eebb8c4ae4b81d5dc11c3a79c4 2024-11-28T01:41:50,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d3c80b68bcc8429bb1944b5da58c092c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d3c80b68bcc8429bb1944b5da58c092c 2024-11-28T01:41:50,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/73ff25fd3fe546519e3bf9518fe5029f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/73ff25fd3fe546519e3bf9518fe5029f 2024-11-28T01:41:50,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c1d470013edd4503838e8f698afa4901 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c1d470013edd4503838e8f698afa4901 2024-11-28T01:41:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/30a80d4f1e5648af9a17210ca8207b96 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/30a80d4f1e5648af9a17210ca8207b96 2024-11-28T01:41:50,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/6d4930b033664edfab454a1989269837 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/6d4930b033664edfab454a1989269837 2024-11-28T01:41:50,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/7743831214954f04a35042d996fe778c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/7743831214954f04a35042d996fe778c 2024-11-28T01:41:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c11beb472d7f44789e00a4f9efc57a27 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/c11beb472d7f44789e00a4f9efc57a27 2024-11-28T01:41:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/64fee9bca22a4dc4b12b3913177dfee8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/64fee9bca22a4dc4b12b3913177dfee8 2024-11-28T01:41:50,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/18ec868a204c44a29bd13b9673ab3d48 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/18ec868a204c44a29bd13b9673ab3d48 2024-11-28T01:41:50,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/74a8e2a54cb446bcbf6ed81b7551a0e9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/74a8e2a54cb446bcbf6ed81b7551a0e9 2024-11-28T01:41:50,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/a4b6c8b7bf844c85a079d970f64fa495 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/a4b6c8b7bf844c85a079d970f64fa495 2024-11-28T01:41:50,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/13a3d836bc09440283e53d061a91bd31 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/13a3d836bc09440283e53d061a91bd31 2024-11-28T01:41:50,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ab93cf8fbe9a4bd7a7bd197bc9b44f44 2024-11-28T01:41:50,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/38de82c8a93a4abdb11e0e1261fdb5e4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/38de82c8a93a4abdb11e0e1261fdb5e4 2024-11-28T01:41:50,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/10cb0a57a0294c98a0fa74d01452e129 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/10cb0a57a0294c98a0fa74d01452e129 2024-11-28T01:41:50,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/069ac031f12d4e56bdc941e2c6dc6670 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/069ac031f12d4e56bdc941e2c6dc6670 2024-11-28T01:41:50,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/cd1c7a97963d48d3bd9deaff81d67ae4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/cd1c7a97963d48d3bd9deaff81d67ae4 2024-11-28T01:41:50,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3303980d8c654ee5a1160c8c8316d7a7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3303980d8c654ee5a1160c8c8316d7a7 2024-11-28T01:41:50,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3c1567c7afbc4a5eaf0c68b8b1d4f617 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3c1567c7afbc4a5eaf0c68b8b1d4f617 2024-11-28T01:41:50,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3643f3657acf415f84e113f6dedd7ab6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/3643f3657acf415f84e113f6dedd7ab6 2024-11-28T01:41:50,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b935379f487e4d9f8cb4fcd1c3bc04bd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/b935379f487e4d9f8cb4fcd1c3bc04bd 2024-11-28T01:41:50,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ca53760a1d4440d895e3f426f3e1cc7e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/ca53760a1d4440d895e3f426f3e1cc7e 2024-11-28T01:41:50,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d50c96264f804dacb4c879bdb4637331 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d50c96264f804dacb4c879bdb4637331 2024-11-28T01:41:50,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8ef75c92b80d4abca6250a4fae46f7fe to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8ef75c92b80d4abca6250a4fae46f7fe 2024-11-28T01:41:50,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d79ef7f1502440628a7eba35e4195f07 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/d79ef7f1502440628a7eba35e4195f07 2024-11-28T01:41:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/0952565ebb354cd2bb3f9078dc3c6b1e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/593138f401bf42e9aa33d95d0c36604c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c35217c61e09437dbdb9733b85a38576, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/68ee0f198bc44e1aaa9dfaedd8a333da, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/feef5acb861147748423ae41e0dbdccf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c08b70a4965342f0a5ffc38ff4570c7f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/114ee791f27549f28da3fea3580f525f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9f32769a4b4a4f4d8e3b077b8aea5d13, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/078c9b050a8f495dbfda4891b6233bc3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/6a824c4824974319bb0263f9c8ac63c4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/77e52b8055f84016bfc2a9d4909ef31b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5c6f8d6162aa4c79a2021197fe78ead5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/4e0696351f2a4183abf585b1e021b720, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/918bed97866a42ed96a32a1839d83e2c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/3e4e86e6b7df41e3bb552ac55ac11038, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/de75be7cd16e4027b70c641ca0b8c2f2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a7261d62b81e435aaca1e8db46df430a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/045ae038263f4857abfe0d4cc3cf6432, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d7c0ee3ac63743d1a77ddf2d086b9eac, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d83bec67bf67487aaa3598fdf13d1286, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/454d078a9b0d48c0b5a39f98dd7e2505, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/93ba26a6f6b641c69856bc0c0ac5a47b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/cdaf70d0b793405da58408ddc91f4878, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d114773d3c1247b98db379ff2a3189f7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a64688e2ef4c4c37905d7c6cc84477ab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/367ab9132fbe45d19b7b5a66c0761dcd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c9705a8b61c44777ba7e92179df55424, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b65c6c04f52a485a926f617a2426ad0a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/220ca155fa824af9adf5efaa2402b6e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9a857f5be04d4ed68c0ed421334c9a29, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c57f39809df843c98db7571f414a642e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b64cce34fdd04940a4e80ef8fa7e1968, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/70718f5e551645f5af79d77d0a9d502a] to archive 2024-11-28T01:41:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:41:50,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/0952565ebb354cd2bb3f9078dc3c6b1e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/0952565ebb354cd2bb3f9078dc3c6b1e 2024-11-28T01:41:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/593138f401bf42e9aa33d95d0c36604c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/593138f401bf42e9aa33d95d0c36604c 2024-11-28T01:41:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c35217c61e09437dbdb9733b85a38576 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c35217c61e09437dbdb9733b85a38576 2024-11-28T01:41:50,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/68ee0f198bc44e1aaa9dfaedd8a333da to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/68ee0f198bc44e1aaa9dfaedd8a333da 2024-11-28T01:41:50,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/feef5acb861147748423ae41e0dbdccf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/feef5acb861147748423ae41e0dbdccf 2024-11-28T01:41:50,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c08b70a4965342f0a5ffc38ff4570c7f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c08b70a4965342f0a5ffc38ff4570c7f 2024-11-28T01:41:50,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/114ee791f27549f28da3fea3580f525f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/114ee791f27549f28da3fea3580f525f 2024-11-28T01:41:50,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9f32769a4b4a4f4d8e3b077b8aea5d13 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9f32769a4b4a4f4d8e3b077b8aea5d13 2024-11-28T01:41:50,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/078c9b050a8f495dbfda4891b6233bc3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/078c9b050a8f495dbfda4891b6233bc3 2024-11-28T01:41:50,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/6a824c4824974319bb0263f9c8ac63c4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/6a824c4824974319bb0263f9c8ac63c4 2024-11-28T01:41:50,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/77e52b8055f84016bfc2a9d4909ef31b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/77e52b8055f84016bfc2a9d4909ef31b 2024-11-28T01:41:50,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5c6f8d6162aa4c79a2021197fe78ead5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5c6f8d6162aa4c79a2021197fe78ead5 2024-11-28T01:41:50,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/4e0696351f2a4183abf585b1e021b720 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/4e0696351f2a4183abf585b1e021b720 2024-11-28T01:41:50,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/918bed97866a42ed96a32a1839d83e2c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/918bed97866a42ed96a32a1839d83e2c 2024-11-28T01:41:50,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/3e4e86e6b7df41e3bb552ac55ac11038 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/3e4e86e6b7df41e3bb552ac55ac11038 2024-11-28T01:41:50,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/de75be7cd16e4027b70c641ca0b8c2f2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/de75be7cd16e4027b70c641ca0b8c2f2 2024-11-28T01:41:50,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a7261d62b81e435aaca1e8db46df430a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a7261d62b81e435aaca1e8db46df430a 2024-11-28T01:41:50,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/045ae038263f4857abfe0d4cc3cf6432 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/045ae038263f4857abfe0d4cc3cf6432 2024-11-28T01:41:50,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d7c0ee3ac63743d1a77ddf2d086b9eac to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d7c0ee3ac63743d1a77ddf2d086b9eac 2024-11-28T01:41:50,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d83bec67bf67487aaa3598fdf13d1286 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d83bec67bf67487aaa3598fdf13d1286 2024-11-28T01:41:50,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/454d078a9b0d48c0b5a39f98dd7e2505 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/454d078a9b0d48c0b5a39f98dd7e2505 2024-11-28T01:41:50,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/93ba26a6f6b641c69856bc0c0ac5a47b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/93ba26a6f6b641c69856bc0c0ac5a47b 2024-11-28T01:41:50,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/cdaf70d0b793405da58408ddc91f4878 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/cdaf70d0b793405da58408ddc91f4878 2024-11-28T01:41:50,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d114773d3c1247b98db379ff2a3189f7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/d114773d3c1247b98db379ff2a3189f7 2024-11-28T01:41:50,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a64688e2ef4c4c37905d7c6cc84477ab to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/a64688e2ef4c4c37905d7c6cc84477ab 2024-11-28T01:41:50,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/367ab9132fbe45d19b7b5a66c0761dcd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/367ab9132fbe45d19b7b5a66c0761dcd 2024-11-28T01:41:50,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c9705a8b61c44777ba7e92179df55424 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c9705a8b61c44777ba7e92179df55424 2024-11-28T01:41:50,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b65c6c04f52a485a926f617a2426ad0a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b65c6c04f52a485a926f617a2426ad0a 2024-11-28T01:41:50,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/220ca155fa824af9adf5efaa2402b6e3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/220ca155fa824af9adf5efaa2402b6e3 2024-11-28T01:41:50,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9a857f5be04d4ed68c0ed421334c9a29 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/9a857f5be04d4ed68c0ed421334c9a29 2024-11-28T01:41:50,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c57f39809df843c98db7571f414a642e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c57f39809df843c98db7571f414a642e 2024-11-28T01:41:50,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b64cce34fdd04940a4e80ef8fa7e1968 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/b64cce34fdd04940a4e80ef8fa7e1968 2024-11-28T01:41:50,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/70718f5e551645f5af79d77d0a9d502a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/70718f5e551645f5af79d77d0a9d502a 2024-11-28T01:41:50,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d50333073f514ee0ac85522f616b4340, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d0e19489ea1f4431b0106c565708ca97, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8cd7f3e405b445f486b1fa1cf27aa089, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/152e0870047040ef90897405ddb8dafc, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/9d1ae3bc78ec4d20a09a30a5b303d33c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/067292ca94cb435c8057f230630dcd1c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/92ddac8bff754916bd163cc1c5f8eebe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/76f682106b8d482ab7d78eb4e16f52a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/761c0d0767414ba9b17864e8d116636d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3009a7469aa5438196f8ddc18701f5c8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3ec64f176162413f9572a8bd0ce443a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4ab6b16f9d3d4d26ab95d77d932091ae, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b884cc031b204f08abacc41bcb326c3b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4f8df85eb3e14e8d9e6a8c474e6396b5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8ddb1098269a49e3a4082843eeac663e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/93e83a86ddad467fa4dad90d43b37c8c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ef15db0e2e4845389f557153f51ca0ab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/23cbd3c9db6a47ab88909bdd88cff13f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/abaa83daa7484a40bb950fec4a826d4f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/caf16c4a9f7d4a69bf130479dfcf328f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2f4cd083626d47a2a63c5b658a62756d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/a2d6728194464e1ebad6f8de337abb8a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/97be92b19680406ca567122a1637d014, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/32e21a792cc5447dac0a468868cffb18, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/150c4020021243b390be2ec503b0ed0a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/77afa8de783c467da125cbbfcf0ad628, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/36d25ae5d9774d3d863789eae7e15d74, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/da96e01a16ea4cbbb1f4fdbab55692d5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/fc442e8914184f27867d0649c6620db7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8fb5489a3dc84c519ffa01e1b07def41, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ce729c690c0444dab5ec0ad4b86e3310, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/f273a9db36f2455dacc9bc3e211df117, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b335825f406a44d19c81ead8d3bf5a51] to archive 2024-11-28T01:41:50,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:41:50,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d50333073f514ee0ac85522f616b4340 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d50333073f514ee0ac85522f616b4340 2024-11-28T01:41:50,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d0e19489ea1f4431b0106c565708ca97 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/d0e19489ea1f4431b0106c565708ca97 2024-11-28T01:41:50,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8cd7f3e405b445f486b1fa1cf27aa089 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8cd7f3e405b445f486b1fa1cf27aa089 2024-11-28T01:41:50,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/152e0870047040ef90897405ddb8dafc to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/152e0870047040ef90897405ddb8dafc 2024-11-28T01:41:50,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/9d1ae3bc78ec4d20a09a30a5b303d33c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/9d1ae3bc78ec4d20a09a30a5b303d33c 2024-11-28T01:41:50,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/067292ca94cb435c8057f230630dcd1c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/067292ca94cb435c8057f230630dcd1c 2024-11-28T01:41:50,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/92ddac8bff754916bd163cc1c5f8eebe to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/92ddac8bff754916bd163cc1c5f8eebe 2024-11-28T01:41:50,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/76f682106b8d482ab7d78eb4e16f52a1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/76f682106b8d482ab7d78eb4e16f52a1 2024-11-28T01:41:50,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/761c0d0767414ba9b17864e8d116636d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/761c0d0767414ba9b17864e8d116636d 2024-11-28T01:41:50,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3009a7469aa5438196f8ddc18701f5c8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3009a7469aa5438196f8ddc18701f5c8 2024-11-28T01:41:50,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3ec64f176162413f9572a8bd0ce443a1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/3ec64f176162413f9572a8bd0ce443a1 2024-11-28T01:41:50,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4ab6b16f9d3d4d26ab95d77d932091ae to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4ab6b16f9d3d4d26ab95d77d932091ae 2024-11-28T01:41:50,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b884cc031b204f08abacc41bcb326c3b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b884cc031b204f08abacc41bcb326c3b 2024-11-28T01:41:50,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4f8df85eb3e14e8d9e6a8c474e6396b5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/4f8df85eb3e14e8d9e6a8c474e6396b5 2024-11-28T01:41:50,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8ddb1098269a49e3a4082843eeac663e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8ddb1098269a49e3a4082843eeac663e 2024-11-28T01:41:50,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/93e83a86ddad467fa4dad90d43b37c8c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/93e83a86ddad467fa4dad90d43b37c8c 2024-11-28T01:41:50,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ef15db0e2e4845389f557153f51ca0ab to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ef15db0e2e4845389f557153f51ca0ab 2024-11-28T01:41:50,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/23cbd3c9db6a47ab88909bdd88cff13f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/23cbd3c9db6a47ab88909bdd88cff13f 2024-11-28T01:41:50,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/abaa83daa7484a40bb950fec4a826d4f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/abaa83daa7484a40bb950fec4a826d4f 2024-11-28T01:41:50,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/caf16c4a9f7d4a69bf130479dfcf328f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/caf16c4a9f7d4a69bf130479dfcf328f 2024-11-28T01:41:50,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2f4cd083626d47a2a63c5b658a62756d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2f4cd083626d47a2a63c5b658a62756d 2024-11-28T01:41:50,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/a2d6728194464e1ebad6f8de337abb8a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/a2d6728194464e1ebad6f8de337abb8a 2024-11-28T01:41:50,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/97be92b19680406ca567122a1637d014 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/97be92b19680406ca567122a1637d014 2024-11-28T01:41:50,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/32e21a792cc5447dac0a468868cffb18 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/32e21a792cc5447dac0a468868cffb18 2024-11-28T01:41:50,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/150c4020021243b390be2ec503b0ed0a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/150c4020021243b390be2ec503b0ed0a 2024-11-28T01:41:50,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/77afa8de783c467da125cbbfcf0ad628 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/77afa8de783c467da125cbbfcf0ad628 2024-11-28T01:41:50,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/36d25ae5d9774d3d863789eae7e15d74 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/36d25ae5d9774d3d863789eae7e15d74 2024-11-28T01:41:50,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/da96e01a16ea4cbbb1f4fdbab55692d5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/da96e01a16ea4cbbb1f4fdbab55692d5 2024-11-28T01:41:50,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/fc442e8914184f27867d0649c6620db7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/fc442e8914184f27867d0649c6620db7 2024-11-28T01:41:50,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8fb5489a3dc84c519ffa01e1b07def41 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/8fb5489a3dc84c519ffa01e1b07def41 2024-11-28T01:41:50,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ce729c690c0444dab5ec0ad4b86e3310 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/ce729c690c0444dab5ec0ad4b86e3310 2024-11-28T01:41:50,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/f273a9db36f2455dacc9bc3e211df117 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/f273a9db36f2455dacc9bc3e211df117 2024-11-28T01:41:50,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b335825f406a44d19c81ead8d3bf5a51 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/b335825f406a44d19c81ead8d3bf5a51 2024-11-28T01:41:50,371 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/recovered.edits/587.seqid, newMaxSeqId=587, maxSeqId=4 2024-11-28T01:41:50,372 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. 2024-11-28T01:41:50,372 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 95ac6ec43a1457c3aa9b4726f69c1394: Waiting for close lock at 1732758108609Running coprocessor pre-close hooks at 1732758108609Disabling compacts and flushes for region at 1732758108609Disabling writes for close at 1732758108609Obtaining lock to block concurrent updates at 1732758108609Preparing flush snapshotting stores in 95ac6ec43a1457c3aa9b4726f69c1394 at 1732758108609Finished memstore snapshotting TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732758108610 (+1 ms)Flushing stores of TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394. at 1732758108610Flushing 95ac6ec43a1457c3aa9b4726f69c1394/A: creating writer at 1732758108610Flushing mob file 95ac6ec43a1457c3aa9b4726f69c1394/A: appending metadata at 1732758108615 (+5 ms)Flushing mob file 95ac6ec43a1457c3aa9b4726f69c1394/A: closing flushed file at 1732758108615Flushing 95ac6ec43a1457c3aa9b4726f69c1394/A: appending metadata at 1732758109025 (+410 ms)Flushing 95ac6ec43a1457c3aa9b4726f69c1394/A: closing flushed file at 1732758109025Flushing 95ac6ec43a1457c3aa9b4726f69c1394/B: creating writer at 1732758109436 (+411 ms)Flushing 95ac6ec43a1457c3aa9b4726f69c1394/B: appending metadata at 1732758109438 (+2 ms)Flushing 95ac6ec43a1457c3aa9b4726f69c1394/B: closing flushed file at 1732758109438Flushing 95ac6ec43a1457c3aa9b4726f69c1394/C: creating writer at 1732758109847 (+409 ms)Flushing 95ac6ec43a1457c3aa9b4726f69c1394/C: appending metadata at 1732758109850 (+3 ms)Flushing 95ac6ec43a1457c3aa9b4726f69c1394/C: closing flushed file at 1732758109850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60785266: reopening flushed file at 1732758110259 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d276971: reopening flushed file at 1732758110263 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5302124b: reopening flushed file at 1732758110268 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 95ac6ec43a1457c3aa9b4726f69c1394 in 1663ms, sequenceid=584, compaction requested=true at 1732758110272 (+4 ms)Writing region close event to WAL at 1732758110369 (+97 ms)Running coprocessor post-close hooks at 1732758110372 (+3 ms)Closed at 1732758110372 2024-11-28T01:41:50,374 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,375 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=95ac6ec43a1457c3aa9b4726f69c1394, regionState=CLOSED 2024-11-28T01:41:50,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:41:50,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-28T01:41:50,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 95ac6ec43a1457c3aa9b4726f69c1394, server=9df3f25f5867,42371,1732758050017 in 1.9210 sec 2024-11-28T01:41:50,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-28T01:41:50,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=95ac6ec43a1457c3aa9b4726f69c1394, UNASSIGN in 1.9260 sec 2024-11-28T01:41:50,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-28T01:41:50,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9290 sec 2024-11-28T01:41:50,383 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758110383"}]},"ts":"1732758110383"} 2024-11-28T01:41:50,384 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T01:41:50,384 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T01:41:50,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9390 sec 2024-11-28T01:41:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T01:41:50,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:50,593 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T01:41:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:50,595 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T01:41:50,596 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:50,600 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,602 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/recovered.edits] 2024-11-28T01:41:50,605 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/00a9a757e8f140678b1cafe4d1c292d4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/00a9a757e8f140678b1cafe4d1c292d4 2024-11-28T01:41:50,606 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/52950c1e900246e7a66d246068eba0bd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/52950c1e900246e7a66d246068eba0bd 2024-11-28T01:41:50,607 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8de053f80bb743ec921cd2559a32b4e2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/A/8de053f80bb743ec921cd2559a32b4e2 2024-11-28T01:41:50,610 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/129fc036e2ff476ab6cf085e37cf7efc to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/129fc036e2ff476ab6cf085e37cf7efc 2024-11-28T01:41:50,611 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5967054a30d04141b71cbf5d8845e261 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/5967054a30d04141b71cbf5d8845e261 2024-11-28T01:41:50,612 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c18ad68074a440589f0c8d9e9efae2d3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/B/c18ad68074a440589f0c8d9e9efae2d3 2024-11-28T01:41:50,614 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2b5e0e30b54d413a98f559783fda5b22 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/2b5e0e30b54d413a98f559783fda5b22 2024-11-28T01:41:50,615 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/965d092b5e9f43d480de884f7bc337a4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/965d092b5e9f43d480de884f7bc337a4 2024-11-28T01:41:50,616 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/cbc172b474134feab7feb0067221ce8b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/C/cbc172b474134feab7feb0067221ce8b 2024-11-28T01:41:50,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/recovered.edits/587.seqid to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394/recovered.edits/587.seqid 2024-11-28T01:41:50,619 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,619 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T01:41:50,619 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T01:41:50,620 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T01:41:50,624 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112802878fd795af486ba4650f6674f51b50_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112802878fd795af486ba4650f6674f51b50_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,625 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808fd12b90776475baf1d0e9b0d752d84_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112808fd12b90776475baf1d0e9b0d752d84_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,626 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112815a52f5131db4071b02df941182e676a_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112815a52f5131db4071b02df941182e676a_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,627 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128240ac90589ae4182a35e7972e5af2c38_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128240ac90589ae4182a35e7972e5af2c38_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,628 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282a2785041c064370a031a1f2f15d7057_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282a2785041c064370a031a1f2f15d7057_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,629 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284990364eaf9d4eba83dbd688a8011c05_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284990364eaf9d4eba83dbd688a8011c05_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,630 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128509fe14096394695bba4555052845ad3_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128509fe14096394695bba4555052845ad3_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,631 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d63561b7e594d74926b5a3d697d7c94_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285d63561b7e594d74926b5a3d697d7c94_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,632 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128651f773bdc064d50a20d883c74123c73_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128651f773bdc064d50a20d883c74123c73_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,633 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112865cfdc068f6742298425d2f4459e74e2_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112865cfdc068f6742298425d2f4459e74e2_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287067c524a3fa4707a59a5ed8b26a2d96_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287067c524a3fa4707a59a5ed8b26a2d96_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112872fb1d0c8b7c487c837635cce74c0c66_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112872fb1d0c8b7c487c837635cce74c0c66_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287b5ecdcd4c78449abd5d2d81a21c8061_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287b5ecdcd4c78449abd5d2d81a21c8061_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128823ea2dd88794853aa8a77f305a19f8c_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128823ea2dd88794853aa8a77f305a19f8c_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883fdc9cfc1af4009b9609e51fc38e4a0_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883fdc9cfc1af4009b9609e51fc38e4a0_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128884c69fc599645c6a2dcfd98d8400497_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128884c69fc599645c6a2dcfd98d8400497_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288bc0788f16f44a748e6a830eddbff14c_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288bc0788f16f44a748e6a830eddbff14c_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128950be47f28e342ca8595addc7346fb8c_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128950be47f28e342ca8595addc7346fb8c_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112897dcd7deaf6848d7a0ebd9a10803f83e_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112897dcd7deaf6848d7a0ebd9a10803f83e_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a1c6c555b9cc4cf29e7e861b91ecc536_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a1c6c555b9cc4cf29e7e861b91ecc536_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b126d38fb2cd4f0b9cdd1c0178a98249_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b126d38fb2cd4f0b9cdd1c0178a98249_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b53a05c688a046e6895c8c662400a400_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b53a05c688a046e6895c8c662400a400_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128becaa7d1cf6e45569161e00201e21a18_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128becaa7d1cf6e45569161e00201e21a18_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128defcad50c28844ac80ef50ca99143a85_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128defcad50c28844ac80ef50ca99143a85_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,652 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e23fac9f497444aeb6495d7e19344efc_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e23fac9f497444aeb6495d7e19344efc_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ff946e873ef44f42af1741cba065c680_95ac6ec43a1457c3aa9b4726f69c1394 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ff946e873ef44f42af1741cba065c680_95ac6ec43a1457c3aa9b4726f69c1394 2024-11-28T01:41:50,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T01:41:50,656 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:50,659 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T01:41:50,662 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T01:41:50,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:50,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T01:41:50,663 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732758110663"}]},"ts":"9223372036854775807"} 2024-11-28T01:41:50,666 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T01:41:50,666 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 95ac6ec43a1457c3aa9b4726f69c1394, NAME => 'TestAcidGuarantees,,1732758078508.95ac6ec43a1457c3aa9b4726f69c1394.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T01:41:50,666 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T01:41:50,666 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732758110666"}]},"ts":"9223372036854775807"} 2024-11-28T01:41:50,670 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T01:41:50,671 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:50,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-28T01:41:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T01:41:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T01:41:50,708 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:50,718 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247 (was 246) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1056209077_22 at /127.0.0.1:42372 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=571 (was 545) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7503 (was 7535) 2024-11-28T01:41:50,732 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=571, ProcessCount=11, AvailableMemoryMB=7501 2024-11-28T01:41:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:41:50,734 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:41:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T01:41:50,736 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:41:50,736 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:50,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-28T01:41:50,737 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:41:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T01:41:50,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742099_1275 (size=1061) 2024-11-28T01:41:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T01:41:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T01:41:51,144 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:41:51,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742100_1276 (size=53) 2024-11-28T01:41:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T01:41:51,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:41:51,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 30f1052f6aa70fa647eb86fc190d7c15, disabling compactions & flushes 2024-11-28T01:41:51,550 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. after waiting 0 ms 2024-11-28T01:41:51,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,550 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30f1052f6aa70fa647eb86fc190d7c15: Waiting for close lock at 1732758111550Disabling compacts and flushes for region at 1732758111550Disabling writes for close at 1732758111550Writing region close event to WAL at 1732758111550Closed at 1732758111550 2024-11-28T01:41:51,551 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:41:51,551 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732758111551"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758111551"}]},"ts":"1732758111551"} 2024-11-28T01:41:51,553 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:41:51,553 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:41:51,554 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758111553"}]},"ts":"1732758111553"} 2024-11-28T01:41:51,555 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T01:41:51,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, ASSIGN}] 2024-11-28T01:41:51,556 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, ASSIGN 2024-11-28T01:41:51,557 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, ASSIGN; state=OFFLINE, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=false 2024-11-28T01:41:51,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=30f1052f6aa70fa647eb86fc190d7c15, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:51,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, ASSIGN because future has completed 2024-11-28T01:41:51,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:41:51,864 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,864 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:41:51,864 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,864 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:41:51,864 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,864 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,866 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,867 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:51,867 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30f1052f6aa70fa647eb86fc190d7c15 columnFamilyName A 2024-11-28T01:41:51,867 DEBUG [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:51,867 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.HStore(327): Store=30f1052f6aa70fa647eb86fc190d7c15/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:51,867 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,868 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:51,868 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30f1052f6aa70fa647eb86fc190d7c15 columnFamilyName B 2024-11-28T01:41:51,868 DEBUG [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:51,869 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.HStore(327): Store=30f1052f6aa70fa647eb86fc190d7c15/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:51,869 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,870 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:41:51,870 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30f1052f6aa70fa647eb86fc190d7c15 columnFamilyName C 2024-11-28T01:41:51,870 DEBUG [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:41:51,870 INFO [StoreOpener-30f1052f6aa70fa647eb86fc190d7c15-1 {}] regionserver.HStore(327): Store=30f1052f6aa70fa647eb86fc190d7c15/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:41:51,870 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,870 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,871 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,871 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,872 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,872 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T01:41:51,872 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:41:51,873 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,875 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:41:51,875 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 30f1052f6aa70fa647eb86fc190d7c15; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62807753, jitterRate=-0.06409154832363129}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:41:51,876 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:51,877 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 30f1052f6aa70fa647eb86fc190d7c15: Running coprocessor pre-open hook at 1732758111864Writing region info on filesystem at 1732758111865 (+1 ms)Initializing all the Stores at 1732758111865Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758111865Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758111865Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758111865Cleaning up temporary data from old regions at 1732758111872 (+7 ms)Running coprocessor post-open hooks at 1732758111876 (+4 ms)Region opened successfully at 1732758111877 (+1 ms) 2024-11-28T01:41:51,878 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., pid=63, masterSystemTime=1732758111861 2024-11-28T01:41:51,879 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,879 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:51,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=30f1052f6aa70fa647eb86fc190d7c15, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:51,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:41:51,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-28T01:41:51,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 in 173 msec 2024-11-28T01:41:51,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-28T01:41:51,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, ASSIGN in 329 msec 2024-11-28T01:41:51,886 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:41:51,886 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758111886"}]},"ts":"1732758111886"} 2024-11-28T01:41:51,887 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T01:41:51,888 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:41:51,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-28T01:41:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-28T01:41:52,883 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:52,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78af2e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,885 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2abfc504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,887 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380c1fcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:41:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,45903,1732758049085 2024-11-28T01:41:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6720058e 2024-11-28T01:41:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:41:52,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:41:52,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,895 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f99e46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64931488, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@942047d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0b9b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22637d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2577651d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38fbe2ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d62e0d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421f0ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2a6e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5aed60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f1073b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@290419ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e40352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f2ac1e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41adfd91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197b1c4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d0fdc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:41:52,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:41:52,923 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:41:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:41:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:41:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25efce0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:41:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:41:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:41:52,924 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:41:52,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72521672, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:41:52,925 DEBUG [Thread-1267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,925 DEBUG [Thread-1272 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,925 DEBUG [Thread-1276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,925 DEBUG [Thread-1280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,925 DEBUG [Thread-1274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,925 DEBUG [Thread-1263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,926 DEBUG [Thread-1278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,926 DEBUG [Thread-1261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,926 DEBUG [Thread-1269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,927 DEBUG [Thread-1265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:41:52,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:52,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:41:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:41:52,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-28T01:41:52,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T01:41:52,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:41:52,932 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:52,933 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:52,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:52,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:41:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:52,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:41:52,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:52,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:53,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732758112980, max timestamp=1732758112999Num uniques 859; 2024-11-28T01:41:53,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:41:53,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758112993, max timestamp=1732758112999Num uniques 859; 2024-11-28T01:41:53,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T01:41:53,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758112993, max timestamp=1732758112999Num uniques 859; 2024-11-28T01:41:53,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T01:41:53,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/68d2a947000c4af8af29572053b0ad2d is 50, key is test_row_0/A:col10/1732758112942/Put/seqid=0 2024-11-28T01:41:53,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742101_1277 (size=12001) 2024-11-28T01:41:53,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/68d2a947000c4af8af29572053b0ad2d 2024-11-28T01:41:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T01:41:53,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758173031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758173037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758173044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758173044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758173046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/a626eb22cd544407ac58945de58175eb is 50, key is test_row_0/B:col10/1732758112942/Put/seqid=0 2024-11-28T01:41:53,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742102_1278 (size=12001) 2024-11-28T01:41:53,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/a626eb22cd544407ac58945de58175eb 2024-11-28T01:41:53,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:53,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:53,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/aafe5007b93449a5b40246f81b6b9b47 is 50, key is test_row_0/C:col10/1732758112942/Put/seqid=0 2024-11-28T01:41:53,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758173152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758173152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758173163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758173163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758173163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742103_1279 (size=12001) 2024-11-28T01:41:53,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T01:41:53,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758173363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758173363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758173372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758173372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758173372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:53,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T01:41:53,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/aafe5007b93449a5b40246f81b6b9b47 2024-11-28T01:41:53,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/68d2a947000c4af8af29572053b0ad2d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/68d2a947000c4af8af29572053b0ad2d 2024-11-28T01:41:53,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/68d2a947000c4af8af29572053b0ad2d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T01:41:53,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/a626eb22cd544407ac58945de58175eb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a626eb22cd544407ac58945de58175eb 2024-11-28T01:41:53,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a626eb22cd544407ac58945de58175eb, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T01:41:53,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/aafe5007b93449a5b40246f81b6b9b47 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/aafe5007b93449a5b40246f81b6b9b47 2024-11-28T01:41:53,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/aafe5007b93449a5b40246f81b6b9b47, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T01:41:53,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=131.15 KB/134302 for 30f1052f6aa70fa647eb86fc190d7c15 in 668ms, sequenceid=13, compaction requested=false 2024-11-28T01:41:53,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:53,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=137.86 KB heapSize=393.31 KB 2024-11-28T01:41:53,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:53,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:53,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:53,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:53,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:53,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:53,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758173685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758173685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758173687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758173687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758173690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/767d637713d34ab889d58c26503dc23b is 50, key is test_row_0/A:col10/1732758113675/Put/seqid=0 2024-11-28T01:41:53,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742104_1280 (size=14341) 2024-11-28T01:41:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758173792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758173792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758173802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758173803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758173803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:53,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:53,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:53,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758174002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758174002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758174013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758174013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758174013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T01:41:54,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/767d637713d34ab889d58c26503dc23b 2024-11-28T01:41:54,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/f38649add60b403d98e1887e7a1c5979 is 50, key is test_row_0/B:col10/1732758113675/Put/seqid=0 2024-11-28T01:41:54,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742105_1281 (size=12001) 2024-11-28T01:41:54,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758174312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758174313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758174323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758174323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758174323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:54,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/f38649add60b403d98e1887e7a1c5979 2024-11-28T01:41:54,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/046e9618c20647fda832cf348d1534e4 is 50, key is test_row_0/C:col10/1732758113675/Put/seqid=0 2024-11-28T01:41:54,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742106_1282 (size=12001) 2024-11-28T01:41:54,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:54,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758174822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758174823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758174833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758174833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758174842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:54,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:54,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:54,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/046e9618c20647fda832cf348d1534e4 2024-11-28T01:41:54,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/767d637713d34ab889d58c26503dc23b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/767d637713d34ab889d58c26503dc23b 2024-11-28T01:41:54,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/767d637713d34ab889d58c26503dc23b, entries=200, sequenceid=50, filesize=14.0 K 2024-11-28T01:41:54,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/f38649add60b403d98e1887e7a1c5979 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/f38649add60b403d98e1887e7a1c5979 2024-11-28T01:41:54,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/f38649add60b403d98e1887e7a1c5979, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T01:41:54,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/046e9618c20647fda832cf348d1534e4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/046e9618c20647fda832cf348d1534e4 2024-11-28T01:41:54,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/046e9618c20647fda832cf348d1534e4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T01:41:54,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~137.86 KB/141172, heapSize ~393.27 KB/402704, currentSize=40.25 KB/41220 for 30f1052f6aa70fa647eb86fc190d7c15 in 1291ms, sequenceid=50, compaction requested=false 2024-11-28T01:41:54,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T01:41:55,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-28T01:41:55,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:55,088 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T01:41:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:55,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/4bca7abc511e47e1bfead0c2cff73a49 is 50, key is test_row_0/A:col10/1732758113677/Put/seqid=0 2024-11-28T01:41:55,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742107_1283 (size=12001) 2024-11-28T01:41:55,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/4bca7abc511e47e1bfead0c2cff73a49 2024-11-28T01:41:55,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e726beb3488f4d7480cc4c5e94b6af5c is 50, key is test_row_0/B:col10/1732758113677/Put/seqid=0 2024-11-28T01:41:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742108_1284 (size=12001) 2024-11-28T01:41:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:55,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:55,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758115861, max timestamp=1732758115867Num uniques 859; 2024-11-28T01:41:55,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:55,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758115861, max timestamp=1732758115867Num uniques 859; 2024-11-28T01:41:55,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:55,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758115861, max timestamp=1732758115867Num uniques 859; 2024-11-28T01:41:55,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:55,895 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:41:55,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758175890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:55,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758175894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:55,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758175895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758175895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758175898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:55,946 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e726beb3488f4d7480cc4c5e94b6af5c 2024-11-28T01:41:55,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/d1fa83079ec8454db908998f501e874c is 50, key is test_row_0/C:col10/1732758113677/Put/seqid=0 2024-11-28T01:41:55,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742109_1285 (size=12001) 2024-11-28T01:41:55,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/d1fa83079ec8454db908998f501e874c 2024-11-28T01:41:55,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/4bca7abc511e47e1bfead0c2cff73a49 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/4bca7abc511e47e1bfead0c2cff73a49 2024-11-28T01:41:55,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/4bca7abc511e47e1bfead0c2cff73a49, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T01:41:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e726beb3488f4d7480cc4c5e94b6af5c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e726beb3488f4d7480cc4c5e94b6af5c 2024-11-28T01:41:56,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758176002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e726beb3488f4d7480cc4c5e94b6af5c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T01:41:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/d1fa83079ec8454db908998f501e874c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d1fa83079ec8454db908998f501e874c 2024-11-28T01:41:56,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758176013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758176013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758176013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758176014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,033 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d1fa83079ec8454db908998f501e874c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T01:41:56,035 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 30f1052f6aa70fa647eb86fc190d7c15 in 947ms, sequenceid=59, compaction requested=true 2024-11-28T01:41:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:56,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-28T01:41:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-28T01:41:56,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-28T01:41:56,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1050 sec 2024-11-28T01:41:56,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 3.1140 sec 2024-11-28T01:41:56,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758115892, max timestamp=1732758116225Num uniques 1068; 2024-11-28T01:41:56,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T01:41:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:56,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758115892, max timestamp=1732758116228Num uniques 1068; 2024-11-28T01:41:56,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T01:41:56,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=73.80 KB heapSize=203.80 KB 2024-11-28T01:41:56,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758115892, max timestamp=1732758116228Num uniques 1068; 2024-11-28T01:41:56,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T01:41:56,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:56,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:56,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:56,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:56,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:56,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:56,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/0aa472e7e0524a11bcc37767a1530dc0 is 50, key is test_row_0/A:col10/1732758116228/Put/seqid=0 2024-11-28T01:41:56,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742110_1286 (size=16681) 2024-11-28T01:41:56,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758116299, max timestamp=1732758116312Num uniques 859; 2024-11-28T01:41:56,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:56,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758116299, max timestamp=1732758116312Num uniques 859; 2024-11-28T01:41:56,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:56,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758116299, max timestamp=1732758116312Num uniques 859; 2024-11-28T01:41:56,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:41:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758176340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758176341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758116336, max timestamp=1732758116345Num uniques 1068; 2024-11-28T01:41:56,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-28T01:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758116336, max timestamp=1732758116345Num uniques 1068; 2024-11-28T01:41:56,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-28T01:41:56,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758116336, max timestamp=1732758116345Num uniques 1068; 2024-11-28T01:41:56,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-28T01:41:56,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732758116375, max timestamp=1732758116396Num uniques 1027; 2024-11-28T01:41:56,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-28T01:41:56,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732758116375, max timestamp=1732758116396Num uniques 1009; 2024-11-28T01:41:56,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732758116375, max timestamp=1732758116396Num uniques 1009; 2024-11-28T01:41:56,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-28T01:41:56,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-28T01:41:56,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732758116426, max timestamp=1732758116446Num uniques 1086; 2024-11-28T01:41:56,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-28T01:41:56,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732758116419, max timestamp=1732758116446Num uniques 1086; 2024-11-28T01:41:56,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-28T01:41:56,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732758116419, max timestamp=1732758116446Num uniques 1086; 2024-11-28T01:41:56,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-28T01:41:56,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732758116461, max timestamp=1732758116480Num uniques 1145; 2024-11-28T01:41:56,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732758116461, max timestamp=1732758116480Num uniques 1145; 2024-11-28T01:41:56,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52430, new segment data size=11210, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-28T01:41:56,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52430, new segment data size=11210, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-28T01:41:56,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1732758116461, max timestamp=1732758116480Num uniques 1045; 2024-11-28T01:41:56,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=11210, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-28T01:41:56,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758176524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1732758116498, max timestamp=1732758116528Num uniques 1104; 2024-11-28T01:41:56,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1732758116498, max timestamp=1732758116528Num uniques 1104; 2024-11-28T01:41:56,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-11-28T01:41:56,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=354, cellSize=16210, totalHeapSize=37056, min timestamp=1732758116498, max timestamp=1732758116528Num uniques 1104; 2024-11-28T01:41:56,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-11-28T01:41:56,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=16210, suffix heap size=142784,new segment heap size=37056 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=354 2024-11-28T01:41:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758176557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758176559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758176562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758176564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758176642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/0aa472e7e0524a11bcc37767a1530dc0 2024-11-28T01:41:56,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5fd626e7b29a444e8c1c4d880deb4a42 is 50, key is test_row_0/B:col10/1732758116228/Put/seqid=0 2024-11-28T01:41:56,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742111_1287 (size=12001) 2024-11-28T01:41:56,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758176673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758176673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758176673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758176673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758176853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758176882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758176882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758176883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758176883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5fd626e7b29a444e8c1c4d880deb4a42 2024-11-28T01:41:57,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/89ae2df4137744c38c7f16c495d9a07d is 50, key is test_row_0/C:col10/1732758116228/Put/seqid=0 2024-11-28T01:41:57,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742112_1288 (size=12001) 2024-11-28T01:41:57,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/89ae2df4137744c38c7f16c495d9a07d 2024-11-28T01:41:57,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/0aa472e7e0524a11bcc37767a1530dc0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0aa472e7e0524a11bcc37767a1530dc0 2024-11-28T01:41:57,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0aa472e7e0524a11bcc37767a1530dc0, entries=250, sequenceid=98, filesize=16.3 K 2024-11-28T01:41:57,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5fd626e7b29a444e8c1c4d880deb4a42 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5fd626e7b29a444e8c1c4d880deb4a42 2024-11-28T01:41:57,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5fd626e7b29a444e8c1c4d880deb4a42, entries=150, sequenceid=98, filesize=11.7 K 2024-11-28T01:41:57,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/89ae2df4137744c38c7f16c495d9a07d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/89ae2df4137744c38c7f16c495d9a07d 2024-11-28T01:41:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T01:41:57,097 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:57,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-28T01:41:57,099 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T01:41:57,100 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:57,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:57,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/89ae2df4137744c38c7f16c495d9a07d, entries=150, sequenceid=98, filesize=11.7 K 2024-11-28T01:41:57,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=161.02 KB/164880 for 30f1052f6aa70fa647eb86fc190d7c15 in 878ms, sequenceid=98, compaction requested=true 2024-11-28T01:41:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:57,112 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:57,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:57,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:57,113 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:57,113 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:57,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:57,113 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:41:57,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:57,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:57,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:57,113 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,113 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/68d2a947000c4af8af29572053b0ad2d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/767d637713d34ab889d58c26503dc23b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/4bca7abc511e47e1bfead0c2cff73a49, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0aa472e7e0524a11bcc37767a1530dc0] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=53.7 K 2024-11-28T01:41:57,114 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68d2a947000c4af8af29572053b0ad2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732758112942 2024-11-28T01:41:57,114 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 767d637713d34ab889d58c26503dc23b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732758113035 2024-11-28T01:41:57,115 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:57,115 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:41:57,115 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,115 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a626eb22cd544407ac58945de58175eb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/f38649add60b403d98e1887e7a1c5979, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e726beb3488f4d7480cc4c5e94b6af5c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5fd626e7b29a444e8c1c4d880deb4a42] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=46.9 K 2024-11-28T01:41:57,115 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bca7abc511e47e1bfead0c2cff73a49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732758113677 2024-11-28T01:41:57,117 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a626eb22cd544407ac58945de58175eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732758112942 2024-11-28T01:41:57,117 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa472e7e0524a11bcc37767a1530dc0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732758115892 2024-11-28T01:41:57,117 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f38649add60b403d98e1887e7a1c5979, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732758113036 2024-11-28T01:41:57,117 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e726beb3488f4d7480cc4c5e94b6af5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732758113677 2024-11-28T01:41:57,118 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd626e7b29a444e8c1c4d880deb4a42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732758115896 2024-11-28T01:41:57,132 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#236 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:57,132 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/3dbfca7599234a669a0ee6628bb0c825 is 50, key is test_row_0/A:col10/1732758116228/Put/seqid=0 2024-11-28T01:41:57,143 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:57,143 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/b67802d02ccc4452b22c33b51627741e is 50, key is test_row_0/B:col10/1732758116228/Put/seqid=0 2024-11-28T01:41:57,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742113_1289 (size=12139) 2024-11-28T01:41:57,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742114_1290 (size=12139) 2024-11-28T01:41:57,170 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/3dbfca7599234a669a0ee6628bb0c825 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/3dbfca7599234a669a0ee6628bb0c825 2024-11-28T01:41:57,170 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/b67802d02ccc4452b22c33b51627741e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/b67802d02ccc4452b22c33b51627741e 2024-11-28T01:41:57,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=167.72 KB heapSize=467.02 KB 2024-11-28T01:41:57,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:41:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:41:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:57,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:41:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:57,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/847df9c86f094efa8926f980acfd2990 is 50, key is test_row_0/A:col10/1732758116561/Put/seqid=0 2024-11-28T01:41:57,180 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into 3dbfca7599234a669a0ee6628bb0c825(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:57,180 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:57,180 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=12, startTime=1732758117111; duration=0sec 2024-11-28T01:41:57,180 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:57,180 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:41:57,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:57,186 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:57,186 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:41:57,186 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,186 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/aafe5007b93449a5b40246f81b6b9b47, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/046e9618c20647fda832cf348d1534e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d1fa83079ec8454db908998f501e874c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/89ae2df4137744c38c7f16c495d9a07d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=46.9 K 2024-11-28T01:41:57,187 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting aafe5007b93449a5b40246f81b6b9b47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732758112942 2024-11-28T01:41:57,187 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 046e9618c20647fda832cf348d1534e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732758113036 2024-11-28T01:41:57,188 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1fa83079ec8454db908998f501e874c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732758113677 2024-11-28T01:41:57,188 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ae2df4137744c38c7f16c495d9a07d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732758115896 2024-11-28T01:41:57,191 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into b67802d02ccc4452b22c33b51627741e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:57,191 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:57,191 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=12, startTime=1732758117113; duration=0sec 2024-11-28T01:41:57,191 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:57,191 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:41:57,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758177193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758177193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758177193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758177193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T01:41:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758177204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,212 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:57,213 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/1d82ecea76074b86ac4f03444815aeba is 50, key is test_row_0/C:col10/1732758116228/Put/seqid=0 2024-11-28T01:41:57,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742115_1291 (size=14541) 2024-11-28T01:41:57,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/847df9c86f094efa8926f980acfd2990 2024-11-28T01:41:57,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9ed8a07116e740e78ec9ff53b90ff639 is 50, key is test_row_0/B:col10/1732758116561/Put/seqid=0 2024-11-28T01:41:57,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742116_1292 (size=12139) 2024-11-28T01:41:57,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T01:41:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,256 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/1d82ecea76074b86ac4f03444815aeba as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/1d82ecea76074b86ac4f03444815aeba 2024-11-28T01:41:57,264 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into 1d82ecea76074b86ac4f03444815aeba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:57,264 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:57,264 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=12, startTime=1732758117113; duration=0sec 2024-11-28T01:41:57,265 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:57,265 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:41:57,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742117_1293 (size=12151) 2024-11-28T01:41:57,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758177312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T01:41:57,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:57,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T01:41:57,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758177522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T01:41:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9ed8a07116e740e78ec9ff53b90ff639 2024-11-28T01:41:57,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/ba026c0620aa4560974a38a64057c85d is 50, key is test_row_0/C:col10/1732758116561/Put/seqid=0 2024-11-28T01:41:57,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742118_1294 (size=12151) 2024-11-28T01:41:57,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/ba026c0620aa4560974a38a64057c85d 2024-11-28T01:41:57,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758177712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758177712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758177712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758177712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T01:41:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:57,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:57,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T01:41:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:57,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/847df9c86f094efa8926f980acfd2990 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/847df9c86f094efa8926f980acfd2990 2024-11-28T01:41:57,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/847df9c86f094efa8926f980acfd2990, entries=200, sequenceid=222, filesize=14.2 K 2024-11-28T01:41:57,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9ed8a07116e740e78ec9ff53b90ff639 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9ed8a07116e740e78ec9ff53b90ff639 2024-11-28T01:41:57,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9ed8a07116e740e78ec9ff53b90ff639, entries=150, sequenceid=222, filesize=11.9 K 2024-11-28T01:41:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/ba026c0620aa4560974a38a64057c85d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ba026c0620aa4560974a38a64057c85d 2024-11-28T01:41:57,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ba026c0620aa4560974a38a64057c85d, entries=150, sequenceid=222, filesize=11.9 K 2024-11-28T01:41:57,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~466.97 KB/478176, currentSize=20.13 KB/20610 for 30f1052f6aa70fa647eb86fc190d7c15 in 576ms, sequenceid=222, compaction requested=false 2024-11-28T01:41:57,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:57,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-28T01:41:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:57,877 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T01:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:57,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:57,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:57,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/0cca3758f96d48418fcf52a741c1a6a3 is 50, key is test_row_0/A:col10/1732758117867/Put/seqid=0 2024-11-28T01:41:57,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742119_1295 (size=12147) 2024-11-28T01:41:57,933 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/0cca3758f96d48418fcf52a741c1a6a3 2024-11-28T01:41:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/c4341560124f415195dfa3e1dacaf93c is 50, key is test_row_0/B:col10/1732758117867/Put/seqid=0 2024-11-28T01:41:57,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742120_1296 (size=9757) 2024-11-28T01:41:57,995 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/c4341560124f415195dfa3e1dacaf93c 2024-11-28T01:41:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/7d79461041f74edc864468da7bc6f676 is 50, key is test_row_0/C:col10/1732758117867/Put/seqid=0 2024-11-28T01:41:58,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742121_1297 (size=9757) 2024-11-28T01:41:58,022 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/7d79461041f74edc864468da7bc6f676 2024-11-28T01:41:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/0cca3758f96d48418fcf52a741c1a6a3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0cca3758f96d48418fcf52a741c1a6a3 2024-11-28T01:41:58,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758117998, max timestamp=1732758118030Num uniques 859; 2024-11-28T01:41:58,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:58,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758117998, max timestamp=1732758118030Num uniques 859; 2024-11-28T01:41:58,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:41:58,042 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0cca3758f96d48418fcf52a741c1a6a3, entries=150, sequenceid=234, filesize=11.9 K 2024-11-28T01:41:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/c4341560124f415195dfa3e1dacaf93c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c4341560124f415195dfa3e1dacaf93c 2024-11-28T01:41:58,050 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c4341560124f415195dfa3e1dacaf93c, entries=100, sequenceid=234, filesize=9.5 K 2024-11-28T01:41:58,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/7d79461041f74edc864468da7bc6f676 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/7d79461041f74edc864468da7bc6f676 2024-11-28T01:41:58,057 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/7d79461041f74edc864468da7bc6f676, entries=100, sequenceid=234, filesize=9.5 K 2024-11-28T01:41:58,057 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=78.27 KB/80150 for 30f1052f6aa70fa647eb86fc190d7c15 in 180ms, sequenceid=234, compaction requested=true 2024-11-28T01:41:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-28T01:41:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-28T01:41:58,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-28T01:41:58,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 959 msec 2024-11-28T01:41:58,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 966 msec 2024-11-28T01:41:58,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=84.98 KB heapSize=243.86 KB 2024-11-28T01:41:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:41:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:58,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/765f00e2542948c88943858fce07660c is 50, key is test_row_0/A:col10/1732758118066/Put/seqid=0 2024-11-28T01:41:58,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742122_1298 (size=14641) 2024-11-28T01:41:58,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/765f00e2542948c88943858fce07660c 2024-11-28T01:41:58,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/c16c83b9f3e1442caede9a235b294f8d is 50, key is test_row_0/B:col10/1732758118066/Put/seqid=0 2024-11-28T01:41:58,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742123_1299 (size=12251) 2024-11-28T01:41:58,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758178196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T01:41:58,233 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:41:58,233 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:41:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-28T01:41:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T01:41:58,235 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:41:58,237 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:41:58,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:41:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758178312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T01:41:58,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T01:41:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758178523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T01:41:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:58,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:58,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:41:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T01:41:58,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/c16c83b9f3e1442caede9a235b294f8d 2024-11-28T01:41:58,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c5768e4c11024f85bd548ca6301fa1c5 is 50, key is test_row_0/C:col10/1732758118066/Put/seqid=0 2024-11-28T01:41:58,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742124_1300 (size=12251) 2024-11-28T01:41:58,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c5768e4c11024f85bd548ca6301fa1c5 2024-11-28T01:41:58,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/765f00e2542948c88943858fce07660c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/765f00e2542948c88943858fce07660c 2024-11-28T01:41:58,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/765f00e2542948c88943858fce07660c, entries=200, sequenceid=259, filesize=14.3 K 2024-11-28T01:41:58,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/c16c83b9f3e1442caede9a235b294f8d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c16c83b9f3e1442caede9a235b294f8d 2024-11-28T01:41:58,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c16c83b9f3e1442caede9a235b294f8d, entries=150, sequenceid=259, filesize=12.0 K 2024-11-28T01:41:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c5768e4c11024f85bd548ca6301fa1c5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c5768e4c11024f85bd548ca6301fa1c5 2024-11-28T01:41:58,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c5768e4c11024f85bd548ca6301fa1c5, entries=150, sequenceid=259, filesize=12.0 K 2024-11-28T01:41:58,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~84.98 KB/87020, heapSize ~243.81 KB/249664, currentSize=93.93 KB/96180 for 30f1052f6aa70fa647eb86fc190d7c15 in 601ms, sequenceid=259, compaction requested=true 2024-11-28T01:41:58,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:58,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:41:58,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:58,670 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:58,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:41:58,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:58,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:41:58,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:41:58,670 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:58,672 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53468 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:58,672 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:58,672 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:41:58,672 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:41:58,672 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,672 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,672 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/3dbfca7599234a669a0ee6628bb0c825, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/847df9c86f094efa8926f980acfd2990, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0cca3758f96d48418fcf52a741c1a6a3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/765f00e2542948c88943858fce07660c] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=52.2 K 2024-11-28T01:41:58,672 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/b67802d02ccc4452b22c33b51627741e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9ed8a07116e740e78ec9ff53b90ff639, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c4341560124f415195dfa3e1dacaf93c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c16c83b9f3e1442caede9a235b294f8d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=45.2 K 2024-11-28T01:41:58,673 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dbfca7599234a669a0ee6628bb0c825, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732758115896 2024-11-28T01:41:58,673 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b67802d02ccc4452b22c33b51627741e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732758115896 2024-11-28T01:41:58,673 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 847df9c86f094efa8926f980acfd2990, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732758116554 2024-11-28T01:41:58,673 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed8a07116e740e78ec9ff53b90ff639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732758116554 2024-11-28T01:41:58,674 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cca3758f96d48418fcf52a741c1a6a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732758117199 2024-11-28T01:41:58,674 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c4341560124f415195dfa3e1dacaf93c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732758117862 2024-11-28T01:41:58,674 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 765f00e2542948c88943858fce07660c, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732758118024 2024-11-28T01:41:58,674 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c16c83b9f3e1442caede9a235b294f8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732758118024 2024-11-28T01:41:58,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T01:41:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:58,699 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T01:41:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:58,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:58,703 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#248 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:58,704 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/fc8180963ad347e7b1618d2d9ea40f57 is 50, key is test_row_0/B:col10/1732758118066/Put/seqid=0 2024-11-28T01:41:58,723 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:58,724 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/a3b4ac3f6ee144f2ae77c61660a358fa is 50, key is test_row_0/A:col10/1732758118066/Put/seqid=0 2024-11-28T01:41:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:58,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:41:58,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/84d6f1bd8b2c4e199775e5740c2d2aa7 is 50, key is test_row_0/A:col10/1732758118106/Put/seqid=0 2024-11-28T01:41:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758178760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758178761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758178764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758178771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742125_1301 (size=12525) 2024-11-28T01:41:58,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742126_1302 (size=12525) 2024-11-28T01:41:58,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742127_1303 (size=12301) 2024-11-28T01:41:58,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758178835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T01:41:58,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758178883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758178883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758178883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758178883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:58,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758179102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758179102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758179102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758179103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,196 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/fc8180963ad347e7b1618d2d9ea40f57 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fc8180963ad347e7b1618d2d9ea40f57 2024-11-28T01:41:59,203 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into fc8180963ad347e7b1618d2d9ea40f57(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:59,203 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:59,203 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=12, startTime=1732758118670; duration=0sec 2024-11-28T01:41:59,203 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:41:59,203 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:41:59,203 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:41:59,205 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:41:59,205 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:41:59,205 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:59,205 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/1d82ecea76074b86ac4f03444815aeba, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ba026c0620aa4560974a38a64057c85d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/7d79461041f74edc864468da7bc6f676, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c5768e4c11024f85bd548ca6301fa1c5] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=45.2 K 2024-11-28T01:41:59,205 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d82ecea76074b86ac4f03444815aeba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732758115896 2024-11-28T01:41:59,206 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ba026c0620aa4560974a38a64057c85d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732758116554 2024-11-28T01:41:59,206 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d79461041f74edc864468da7bc6f676, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732758117862 2024-11-28T01:41:59,207 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c5768e4c11024f85bd548ca6301fa1c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732758118024 2024-11-28T01:41:59,211 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/a3b4ac3f6ee144f2ae77c61660a358fa as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a3b4ac3f6ee144f2ae77c61660a358fa 2024-11-28T01:41:59,217 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into a3b4ac3f6ee144f2ae77c61660a358fa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:59,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:59,217 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=12, startTime=1732758118670; duration=0sec 2024-11-28T01:41:59,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:59,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:41:59,235 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:41:59,235 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/924a80a903394732810b094e32c8b92b is 50, key is test_row_0/C:col10/1732758118066/Put/seqid=0 2024-11-28T01:41:59,237 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/84d6f1bd8b2c4e199775e5740c2d2aa7 2024-11-28T01:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/4cd0ec8feb82455ab4839fc32639807d is 50, key is test_row_0/B:col10/1732758118106/Put/seqid=0 2024-11-28T01:41:59,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742128_1304 (size=12525) 2024-11-28T01:41:59,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742129_1305 (size=12301) 2024-11-28T01:41:59,329 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/4cd0ec8feb82455ab4839fc32639807d 2024-11-28T01:41:59,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/24afa6247c114c82a284fcb412cffa63 is 50, key is test_row_0/C:col10/1732758118106/Put/seqid=0 2024-11-28T01:41:59,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758179353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T01:41:59,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742130_1306 (size=12301) 2024-11-28T01:41:59,382 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/24afa6247c114c82a284fcb412cffa63 2024-11-28T01:41:59,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/84d6f1bd8b2c4e199775e5740c2d2aa7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/84d6f1bd8b2c4e199775e5740c2d2aa7 2024-11-28T01:41:59,392 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/84d6f1bd8b2c4e199775e5740c2d2aa7, entries=150, sequenceid=276, filesize=12.0 K 2024-11-28T01:41:59,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/4cd0ec8feb82455ab4839fc32639807d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/4cd0ec8feb82455ab4839fc32639807d 2024-11-28T01:41:59,399 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/4cd0ec8feb82455ab4839fc32639807d, entries=150, sequenceid=276, filesize=12.0 K 2024-11-28T01:41:59,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/24afa6247c114c82a284fcb412cffa63 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/24afa6247c114c82a284fcb412cffa63 2024-11-28T01:41:59,404 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/24afa6247c114c82a284fcb412cffa63, entries=150, sequenceid=276, filesize=12.0 K 2024-11-28T01:41:59,405 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 30f1052f6aa70fa647eb86fc190d7c15 in 705ms, sequenceid=276, compaction requested=false 2024-11-28T01:41:59,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:59,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:41:59,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-28T01:41:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-28T01:41:59,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-28T01:41:59,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1690 sec 2024-11-28T01:41:59,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1760 sec 2024-11-28T01:41:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:41:59,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T01:41:59,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:41:59,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:59,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:41:59,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:59,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:41:59,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:41:59,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1cf96189796f4bfe9bfb3e2de8ebb591 is 50, key is test_row_0/A:col10/1732758118762/Put/seqid=0 2024-11-28T01:41:59,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758179443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758179445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758179453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758179451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742131_1307 (size=17181) 2024-11-28T01:41:59,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758179562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758179563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758179573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758179573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,704 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/924a80a903394732810b094e32c8b92b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/924a80a903394732810b094e32c8b92b 2024-11-28T01:41:59,714 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into 924a80a903394732810b094e32c8b92b(size=12.2 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:41:59,714 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:41:59,714 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=12, startTime=1732758118670; duration=0sec 2024-11-28T01:41:59,715 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:41:59,715 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:41:59,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758179774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758179774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758179782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:41:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758179782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:41:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:41:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:41:59,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1cf96189796f4bfe9bfb3e2de8ebb591 2024-11-28T01:41:59,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0116bf4a20d84c9b9fb4e60e58632487 is 50, key is test_row_0/B:col10/1732758118762/Put/seqid=0 2024-11-28T01:41:59,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742132_1308 (size=12301) 2024-11-28T01:41:59,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0116bf4a20d84c9b9fb4e60e58632487 2024-11-28T01:41:59,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/0bcda7da91cb4764a0ee71ff84311ef4 is 50, key is test_row_0/C:col10/1732758118762/Put/seqid=0 2024-11-28T01:41:59,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742133_1309 (size=12301) 2024-11-28T01:41:59,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/0bcda7da91cb4764a0ee71ff84311ef4 2024-11-28T01:41:59,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1cf96189796f4bfe9bfb3e2de8ebb591 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1cf96189796f4bfe9bfb3e2de8ebb591 2024-11-28T01:42:00,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1cf96189796f4bfe9bfb3e2de8ebb591, entries=250, sequenceid=296, filesize=16.8 K 2024-11-28T01:42:00,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0116bf4a20d84c9b9fb4e60e58632487 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0116bf4a20d84c9b9fb4e60e58632487 2024-11-28T01:42:00,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0116bf4a20d84c9b9fb4e60e58632487, entries=150, sequenceid=296, filesize=12.0 K 2024-11-28T01:42:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/0bcda7da91cb4764a0ee71ff84311ef4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/0bcda7da91cb4764a0ee71ff84311ef4 2024-11-28T01:42:00,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/0bcda7da91cb4764a0ee71ff84311ef4, entries=150, sequenceid=296, filesize=12.0 K 2024-11-28T01:42:00,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 30f1052f6aa70fa647eb86fc190d7c15 in 611ms, sequenceid=296, compaction requested=true 2024-11-28T01:42:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:00,030 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:00,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:42:00,030 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:00,033 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:00,033 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:00,033 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:00,033 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:00,033 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,033 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,033 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fc8180963ad347e7b1618d2d9ea40f57, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/4cd0ec8feb82455ab4839fc32639807d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0116bf4a20d84c9b9fb4e60e58632487] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.3 K 2024-11-28T01:42:00,033 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a3b4ac3f6ee144f2ae77c61660a358fa, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/84d6f1bd8b2c4e199775e5740c2d2aa7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1cf96189796f4bfe9bfb3e2de8ebb591] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=41.0 K 2024-11-28T01:42:00,034 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting fc8180963ad347e7b1618d2d9ea40f57, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732758118024 2024-11-28T01:42:00,034 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd0ec8feb82455ab4839fc32639807d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732758118106 2024-11-28T01:42:00,034 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b4ac3f6ee144f2ae77c61660a358fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732758118024 2024-11-28T01:42:00,034 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0116bf4a20d84c9b9fb4e60e58632487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732758118762 2024-11-28T01:42:00,035 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84d6f1bd8b2c4e199775e5740c2d2aa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732758118106 2024-11-28T01:42:00,035 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf96189796f4bfe9bfb3e2de8ebb591, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732758118762 2024-11-28T01:42:00,054 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:00,055 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/86fcda6a773847b3a01094e8d7c9efc3 is 50, key is test_row_0/A:col10/1732758118762/Put/seqid=0 2024-11-28T01:42:00,058 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:00,059 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/44773caf793c4a9581176764d4e9dd16 is 50, key is test_row_0/B:col10/1732758118762/Put/seqid=0 2024-11-28T01:42:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:00,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T01:42:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:00,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742134_1310 (size=12677) 2024-11-28T01:42:00,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/a2ab7bfe5db74f3398fa7aa9fc116ec0 is 50, key is test_row_0/A:col10/1732758120085/Put/seqid=0 2024-11-28T01:42:00,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742135_1311 (size=12677) 2024-11-28T01:42:00,103 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/86fcda6a773847b3a01094e8d7c9efc3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/86fcda6a773847b3a01094e8d7c9efc3 2024-11-28T01:42:00,110 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into 86fcda6a773847b3a01094e8d7c9efc3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:00,110 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:00,110 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758120030; duration=0sec 2024-11-28T01:42:00,111 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:00,111 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:00,111 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:00,112 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:00,112 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:00,112 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,112 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/924a80a903394732810b094e32c8b92b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/24afa6247c114c82a284fcb412cffa63, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/0bcda7da91cb4764a0ee71ff84311ef4] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.3 K 2024-11-28T01:42:00,113 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 924a80a903394732810b094e32c8b92b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732758118024 2024-11-28T01:42:00,114 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24afa6247c114c82a284fcb412cffa63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732758118106 2024-11-28T01:42:00,115 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcda7da91cb4764a0ee71ff84311ef4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732758118762 2024-11-28T01:42:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758180120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758180121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758180128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758180129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742136_1312 (size=12301) 2024-11-28T01:42:00,138 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:00,139 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/4c63a08d13224c7fb8d2e96ff897c5af is 50, key is test_row_0/C:col10/1732758118762/Put/seqid=0 2024-11-28T01:42:00,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/a2ab7bfe5db74f3398fa7aa9fc116ec0 2024-11-28T01:42:00,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/830e39eea8794f96b5ed6b6849555b3f is 50, key is test_row_0/B:col10/1732758120085/Put/seqid=0 2024-11-28T01:42:00,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742137_1313 (size=12677) 2024-11-28T01:42:00,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/4c63a08d13224c7fb8d2e96ff897c5af as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c63a08d13224c7fb8d2e96ff897c5af 2024-11-28T01:42:00,195 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into 4c63a08d13224c7fb8d2e96ff897c5af(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:00,195 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:00,195 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=13, startTime=1732758120030; duration=0sec 2024-11-28T01:42:00,195 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:00,195 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:00,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742138_1314 (size=12301) 2024-11-28T01:42:00,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/830e39eea8794f96b5ed6b6849555b3f 2024-11-28T01:42:00,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/190cea53ba174682ab8ab83dfb7e8f15 is 50, key is test_row_0/C:col10/1732758120085/Put/seqid=0 2024-11-28T01:42:00,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758180233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758180233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742139_1315 (size=12301) 2024-11-28T01:42:00,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758180242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/190cea53ba174682ab8ab83dfb7e8f15 2024-11-28T01:42:00,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758180243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/a2ab7bfe5db74f3398fa7aa9fc116ec0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a2ab7bfe5db74f3398fa7aa9fc116ec0 2024-11-28T01:42:00,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a2ab7bfe5db74f3398fa7aa9fc116ec0, entries=150, sequenceid=312, filesize=12.0 K 2024-11-28T01:42:00,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/830e39eea8794f96b5ed6b6849555b3f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/830e39eea8794f96b5ed6b6849555b3f 2024-11-28T01:42:00,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/830e39eea8794f96b5ed6b6849555b3f, entries=150, sequenceid=312, filesize=12.0 K 2024-11-28T01:42:00,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/190cea53ba174682ab8ab83dfb7e8f15 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/190cea53ba174682ab8ab83dfb7e8f15 2024-11-28T01:42:00,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/190cea53ba174682ab8ab83dfb7e8f15, entries=150, sequenceid=312, filesize=12.0 K 2024-11-28T01:42:00,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 30f1052f6aa70fa647eb86fc190d7c15 in 192ms, sequenceid=312, compaction requested=false 2024-11-28T01:42:00,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:00,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:42:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T01:42:00,383 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:00,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-28T01:42:00,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T01:42:00,387 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:00,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:00,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1bc3eca21a004184bd1f8215f7220114 is 50, key is test_row_0/A:col10/1732758120115/Put/seqid=0 2024-11-28T01:42:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758180447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758180449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758180451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758180453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758180453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742140_1316 (size=14741) 2024-11-28T01:42:00,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1bc3eca21a004184bd1f8215f7220114 2024-11-28T01:42:00,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/347106f8e62d4cd2a8fff3bf8796a9e9 is 50, key is test_row_0/B:col10/1732758120115/Put/seqid=0 2024-11-28T01:42:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T01:42:00,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742141_1317 (size=12301) 2024-11-28T01:42:00,508 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/44773caf793c4a9581176764d4e9dd16 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/44773caf793c4a9581176764d4e9dd16 2024-11-28T01:42:00,514 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 44773caf793c4a9581176764d4e9dd16(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:00,514 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:00,514 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758120030; duration=0sec 2024-11-28T01:42:00,514 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:00,514 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:00,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T01:42:00,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758180563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758180563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758180563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T01:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T01:42:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758180762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758180763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758180773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758180773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758180773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:00,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T01:42:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:00,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:00,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/347106f8e62d4cd2a8fff3bf8796a9e9 2024-11-28T01:42:00,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/9912b9c5992b41498c3ff3c56e1d0a91 is 50, key is test_row_0/C:col10/1732758120115/Put/seqid=0 2024-11-28T01:42:00,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742142_1318 (size=12301) 2024-11-28T01:42:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T01:42:01,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T01:42:01,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:01,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758181082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758181082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758181082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T01:42:01,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758181272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758181272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T01:42:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:01,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/9912b9c5992b41498c3ff3c56e1d0a91 2024-11-28T01:42:01,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1bc3eca21a004184bd1f8215f7220114 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1bc3eca21a004184bd1f8215f7220114 2024-11-28T01:42:01,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1bc3eca21a004184bd1f8215f7220114, entries=200, sequenceid=333, filesize=14.4 K 2024-11-28T01:42:01,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/347106f8e62d4cd2a8fff3bf8796a9e9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/347106f8e62d4cd2a8fff3bf8796a9e9 2024-11-28T01:42:01,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/347106f8e62d4cd2a8fff3bf8796a9e9, entries=150, sequenceid=333, filesize=12.0 K 2024-11-28T01:42:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/9912b9c5992b41498c3ff3c56e1d0a91 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9912b9c5992b41498c3ff3c56e1d0a91 2024-11-28T01:42:01,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9912b9c5992b41498c3ff3c56e1d0a91, entries=150, sequenceid=333, filesize=12.0 K 2024-11-28T01:42:01,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 30f1052f6aa70fa647eb86fc190d7c15 in 1036ms, sequenceid=333, compaction requested=true 2024-11-28T01:42:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:01,411 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:01,411 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:01,412 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:01,412 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:01,412 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:01,412 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:01,412 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,413 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/86fcda6a773847b3a01094e8d7c9efc3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a2ab7bfe5db74f3398fa7aa9fc116ec0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1bc3eca21a004184bd1f8215f7220114] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=38.8 K 2024-11-28T01:42:01,413 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,413 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/44773caf793c4a9581176764d4e9dd16, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/830e39eea8794f96b5ed6b6849555b3f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/347106f8e62d4cd2a8fff3bf8796a9e9] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.4 K 2024-11-28T01:42:01,413 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 44773caf793c4a9581176764d4e9dd16, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732758118762 2024-11-28T01:42:01,413 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86fcda6a773847b3a01094e8d7c9efc3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732758118762 2024-11-28T01:42:01,414 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 830e39eea8794f96b5ed6b6849555b3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732758119449 2024-11-28T01:42:01,414 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ab7bfe5db74f3398fa7aa9fc116ec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732758119449 2024-11-28T01:42:01,414 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 347106f8e62d4cd2a8fff3bf8796a9e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732758120115 2024-11-28T01:42:01,414 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc3eca21a004184bd1f8215f7220114, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732758120115 2024-11-28T01:42:01,424 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:01,424 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/934b801c92454108a852ba1da43d1a91 is 50, key is test_row_0/B:col10/1732758120115/Put/seqid=0 2024-11-28T01:42:01,426 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:01,427 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/67fea92c8c7c4753a3ca457765218266 is 50, key is test_row_0/A:col10/1732758120115/Put/seqid=0 2024-11-28T01:42:01,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742144_1320 (size=12779) 2024-11-28T01:42:01,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T01:42:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,477 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:42:01,477 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/934b801c92454108a852ba1da43d1a91 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/934b801c92454108a852ba1da43d1a91 2024-11-28T01:42:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:01,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742143_1319 (size=12779) 2024-11-28T01:42:01,484 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 934b801c92454108a852ba1da43d1a91(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:01,484 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:01,484 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758121411; duration=0sec 2024-11-28T01:42:01,484 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:01,484 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:01,485 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:01,486 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:01,486 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:01,486 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:01,486 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c63a08d13224c7fb8d2e96ff897c5af, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/190cea53ba174682ab8ab83dfb7e8f15, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9912b9c5992b41498c3ff3c56e1d0a91] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.4 K 2024-11-28T01:42:01,488 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c63a08d13224c7fb8d2e96ff897c5af, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732758118762 2024-11-28T01:42:01,488 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 190cea53ba174682ab8ab83dfb7e8f15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732758119449 2024-11-28T01:42:01,488 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9912b9c5992b41498c3ff3c56e1d0a91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732758120115 2024-11-28T01:42:01,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e4e354a5860f49a58667a056247f4c15 is 50, key is test_row_0/A:col10/1732758120434/Put/seqid=0 2024-11-28T01:42:01,503 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/67fea92c8c7c4753a3ca457765218266 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/67fea92c8c7c4753a3ca457765218266 2024-11-28T01:42:01,509 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into 67fea92c8c7c4753a3ca457765218266(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:01,509 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:01,509 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758121411; duration=0sec 2024-11-28T01:42:01,509 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:01,509 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:01,512 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:01,513 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/cb1971b1647b44ce8af320f2bbf87a39 is 50, key is test_row_0/C:col10/1732758120115/Put/seqid=0 2024-11-28T01:42:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T01:42:01,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742145_1321 (size=9857) 2024-11-28T01:42:01,549 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e4e354a5860f49a58667a056247f4c15 2024-11-28T01:42:01,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742146_1322 (size=12779) 2024-11-28T01:42:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/bbe71b40b9c8490d8ebfca4d2903a799 is 50, key is test_row_0/B:col10/1732758120434/Put/seqid=0 2024-11-28T01:42:01,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742147_1323 (size=9857) 2024-11-28T01:42:01,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:01,599 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/bbe71b40b9c8490d8ebfca4d2903a799 2024-11-28T01:42:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/07166c8a13f741e8b9d0483e3020c5fb is 50, key is test_row_0/C:col10/1732758120434/Put/seqid=0 2024-11-28T01:42:01,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742148_1324 (size=9857) 2024-11-28T01:42:01,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758181634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758181635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758181636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758181742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758181743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758181753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758181953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758181953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758181962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:01,970 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/cb1971b1647b44ce8af320f2bbf87a39 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/cb1971b1647b44ce8af320f2bbf87a39 2024-11-28T01:42:01,976 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into cb1971b1647b44ce8af320f2bbf87a39(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:01,976 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:01,976 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=13, startTime=1732758121411; duration=0sec 2024-11-28T01:42:01,976 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:01,976 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:02,036 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/07166c8a13f741e8b9d0483e3020c5fb 2024-11-28T01:42:02,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e4e354a5860f49a58667a056247f4c15 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e4e354a5860f49a58667a056247f4c15 2024-11-28T01:42:02,047 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e4e354a5860f49a58667a056247f4c15, entries=100, sequenceid=347, filesize=9.6 K 2024-11-28T01:42:02,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/bbe71b40b9c8490d8ebfca4d2903a799 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/bbe71b40b9c8490d8ebfca4d2903a799 2024-11-28T01:42:02,054 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/bbe71b40b9c8490d8ebfca4d2903a799, entries=100, sequenceid=347, filesize=9.6 K 2024-11-28T01:42:02,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/07166c8a13f741e8b9d0483e3020c5fb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/07166c8a13f741e8b9d0483e3020c5fb 2024-11-28T01:42:02,061 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/07166c8a13f741e8b9d0483e3020c5fb, entries=100, sequenceid=347, filesize=9.6 K 2024-11-28T01:42:02,062 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 30f1052f6aa70fa647eb86fc190d7c15 in 585ms, sequenceid=347, compaction requested=false 2024-11-28T01:42:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:02,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-28T01:42:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-28T01:42:02,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-28T01:42:02,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-28T01:42:02,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6850 sec 2024-11-28T01:42:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:02,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T01:42:02,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:02,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:02,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:02,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:02,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:02,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:02,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/7f4d446686a443dd96454ddb11f00571 is 50, key is test_row_0/A:col10/1732758121633/Put/seqid=0 2024-11-28T01:42:02,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742149_1325 (size=17181) 2024-11-28T01:42:02,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758182288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758182289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758182289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758182290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758182292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,299 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 30f1052f6aa70fa647eb86fc190d7c15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:42:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758182403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758182403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758182403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758182403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T01:42:02,533 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:02,534 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-28T01:42:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T01:42:02,541 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:02,542 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:02,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:02,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758182613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758182613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758182613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758182613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T01:42:02,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/7f4d446686a443dd96454ddb11f00571 2024-11-28T01:42:02,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T01:42:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:02,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:02,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:02,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/1024caff7ccf46bea5bce7bcc885360a is 50, key is test_row_0/B:col10/1732758121633/Put/seqid=0 2024-11-28T01:42:02,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742150_1326 (size=12301) 2024-11-28T01:42:02,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/1024caff7ccf46bea5bce7bcc885360a 2024-11-28T01:42:02,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c28132a4f9e3437fbf504846abce43a1 is 50, key is test_row_0/C:col10/1732758121633/Put/seqid=0 2024-11-28T01:42:02,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742151_1327 (size=12301) 2024-11-28T01:42:02,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c28132a4f9e3437fbf504846abce43a1 2024-11-28T01:42:02,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/7f4d446686a443dd96454ddb11f00571 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7f4d446686a443dd96454ddb11f00571 2024-11-28T01:42:02,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7f4d446686a443dd96454ddb11f00571, entries=250, sequenceid=372, filesize=16.8 K 2024-11-28T01:42:02,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/1024caff7ccf46bea5bce7bcc885360a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1024caff7ccf46bea5bce7bcc885360a 2024-11-28T01:42:02,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1024caff7ccf46bea5bce7bcc885360a, entries=150, sequenceid=372, filesize=12.0 K 2024-11-28T01:42:02,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c28132a4f9e3437fbf504846abce43a1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c28132a4f9e3437fbf504846abce43a1 2024-11-28T01:42:02,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c28132a4f9e3437fbf504846abce43a1, entries=150, sequenceid=372, filesize=12.0 K 2024-11-28T01:42:02,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=60.38 KB/61830 for 30f1052f6aa70fa647eb86fc190d7c15 in 512ms, sequenceid=372, compaction requested=true 2024-11-28T01:42:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:02,776 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:02,776 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:02,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:02,777 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:02,777 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39817 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:02,778 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:02,778 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:02,778 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:02,778 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:02,778 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/67fea92c8c7c4753a3ca457765218266, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e4e354a5860f49a58667a056247f4c15, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7f4d446686a443dd96454ddb11f00571] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=38.9 K 2024-11-28T01:42:02,778 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/934b801c92454108a852ba1da43d1a91, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/bbe71b40b9c8490d8ebfca4d2903a799, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1024caff7ccf46bea5bce7bcc885360a] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=34.1 K 2024-11-28T01:42:02,779 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 934b801c92454108a852ba1da43d1a91, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732758120115 2024-11-28T01:42:02,779 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67fea92c8c7c4753a3ca457765218266, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732758120115 2024-11-28T01:42:02,779 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe71b40b9c8490d8ebfca4d2903a799, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732758120434 2024-11-28T01:42:02,779 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e354a5860f49a58667a056247f4c15, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732758120434 2024-11-28T01:42:02,779 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1024caff7ccf46bea5bce7bcc885360a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732758121633 2024-11-28T01:42:02,780 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4d446686a443dd96454ddb11f00571, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732758121624 2024-11-28T01:42:02,796 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:02,797 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/03b3e84e9d9a4ab2a616be4394a86b2b is 50, key is test_row_0/A:col10/1732758121633/Put/seqid=0 2024-11-28T01:42:02,801 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:02,801 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9a90103e8330478a9f5118a5e0aebff3 is 50, key is test_row_0/B:col10/1732758121633/Put/seqid=0 2024-11-28T01:42:02,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742152_1328 (size=12881) 2024-11-28T01:42:02,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742153_1329 (size=12881) 2024-11-28T01:42:02,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T01:42:02,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:02,850 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:42:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T01:42:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e991f00010b04117bd5fc5ab4311af59 is 50, key is test_row_0/A:col10/1732758122287/Put/seqid=0 2024-11-28T01:42:02,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742154_1330 (size=12301) 2024-11-28T01:42:02,892 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e991f00010b04117bd5fc5ab4311af59 2024-11-28T01:42:02,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/a771652818e54a1c99698d008224a50d is 50, key is test_row_0/B:col10/1732758122287/Put/seqid=0 2024-11-28T01:42:02,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742155_1331 (size=12301) 2024-11-28T01:42:02,923 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/a771652818e54a1c99698d008224a50d 2024-11-28T01:42:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:02,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:02,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/11400d0c42fb47c4b1f57fff1d627eff is 50, key is test_row_0/C:col10/1732758122287/Put/seqid=0 2024-11-28T01:42:02,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742156_1332 (size=12301) 2024-11-28T01:42:02,963 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/11400d0c42fb47c4b1f57fff1d627eff 2024-11-28T01:42:02,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e991f00010b04117bd5fc5ab4311af59 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e991f00010b04117bd5fc5ab4311af59 2024-11-28T01:42:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758182966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758182969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e991f00010b04117bd5fc5ab4311af59, entries=150, sequenceid=384, filesize=12.0 K 2024-11-28T01:42:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/a771652818e54a1c99698d008224a50d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a771652818e54a1c99698d008224a50d 2024-11-28T01:42:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758182974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758182975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:02,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a771652818e54a1c99698d008224a50d, entries=150, sequenceid=384, filesize=12.0 K 2024-11-28T01:42:02,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/11400d0c42fb47c4b1f57fff1d627eff as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/11400d0c42fb47c4b1f57fff1d627eff 2024-11-28T01:42:02,997 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/11400d0c42fb47c4b1f57fff1d627eff, entries=150, sequenceid=384, filesize=12.0 K 2024-11-28T01:42:02,998 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 30f1052f6aa70fa647eb86fc190d7c15 in 148ms, sequenceid=384, compaction requested=true 2024-11-28T01:42:02,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:02,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:02,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-28T01:42:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-28T01:42:03,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-28T01:42:03,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 457 msec 2024-11-28T01:42:03,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 469 msec 2024-11-28T01:42:03,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T01:42:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:03,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758122954, max timestamp=1732758123086Num uniques 859; 2024-11-28T01:42:03,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:42:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:03,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:03,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:03,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:03,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:03,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:03,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/f1ad0d8246b84823aabb4d03adfcd1ea is 50, key is test_row_0/A:col10/1732758123089/Put/seqid=0 2024-11-28T01:42:03,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758183124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758183135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758183137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758183138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742157_1333 (size=12301) 2024-11-28T01:42:03,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/f1ad0d8246b84823aabb4d03adfcd1ea 2024-11-28T01:42:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T01:42:03,162 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:03,163 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-28T01:42:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T01:42:03,166 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:03,167 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:03,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:03,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5faaed00f39c4622aa9bb613fc04acbf is 50, key is test_row_0/B:col10/1732758123089/Put/seqid=0 2024-11-28T01:42:03,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742158_1334 (size=12301) 2024-11-28T01:42:03,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5faaed00f39c4622aa9bb613fc04acbf 2024-11-28T01:42:03,229 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/03b3e84e9d9a4ab2a616be4394a86b2b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/03b3e84e9d9a4ab2a616be4394a86b2b 2024-11-28T01:42:03,232 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9a90103e8330478a9f5118a5e0aebff3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9a90103e8330478a9f5118a5e0aebff3 2024-11-28T01:42:03,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/836ddcbf43384bd1b4b9f7974182d877 is 50, key is test_row_0/C:col10/1732758123089/Put/seqid=0 2024-11-28T01:42:03,240 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into 03b3e84e9d9a4ab2a616be4394a86b2b(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:03,240 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:03,240 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758122776; duration=0sec 2024-11-28T01:42:03,240 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:03,240 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:03,241 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:03,245 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47238 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:03,245 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:03,245 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:03,245 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/cb1971b1647b44ce8af320f2bbf87a39, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/07166c8a13f741e8b9d0483e3020c5fb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c28132a4f9e3437fbf504846abce43a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/11400d0c42fb47c4b1f57fff1d627eff] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=46.1 K 2024-11-28T01:42:03,246 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1971b1647b44ce8af320f2bbf87a39, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732758120115 2024-11-28T01:42:03,246 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07166c8a13f741e8b9d0483e3020c5fb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732758120434 2024-11-28T01:42:03,247 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c28132a4f9e3437fbf504846abce43a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732758121633 2024-11-28T01:42:03,248 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11400d0c42fb47c4b1f57fff1d627eff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732758122286 2024-11-28T01:42:03,248 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 9a90103e8330478a9f5118a5e0aebff3(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:03,248 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:03,248 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758122776; duration=0sec 2024-11-28T01:42:03,248 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:03,248 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742159_1335 (size=12301) 2024-11-28T01:42:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758183243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/836ddcbf43384bd1b4b9f7974182d877 2024-11-28T01:42:03,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/f1ad0d8246b84823aabb4d03adfcd1ea as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f1ad0d8246b84823aabb4d03adfcd1ea 2024-11-28T01:42:03,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758183252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,259 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:03,260 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/b6cca1209b0948e0bfa196185a1b9501 is 50, key is test_row_0/C:col10/1732758122287/Put/seqid=0 2024-11-28T01:42:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758183253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f1ad0d8246b84823aabb4d03adfcd1ea, entries=150, sequenceid=406, filesize=12.0 K 2024-11-28T01:42:03,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5faaed00f39c4622aa9bb613fc04acbf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5faaed00f39c4622aa9bb613fc04acbf 2024-11-28T01:42:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758183252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5faaed00f39c4622aa9bb613fc04acbf, entries=150, sequenceid=406, filesize=12.0 K 2024-11-28T01:42:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T01:42:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/836ddcbf43384bd1b4b9f7974182d877 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/836ddcbf43384bd1b4b9f7974182d877 2024-11-28T01:42:03,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742160_1336 (size=12915) 2024-11-28T01:42:03,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/836ddcbf43384bd1b4b9f7974182d877, entries=150, sequenceid=406, filesize=12.0 K 2024-11-28T01:42:03,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for 30f1052f6aa70fa647eb86fc190d7c15 in 198ms, sequenceid=406, compaction requested=true 2024-11-28T01:42:03,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:03,289 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:03,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:42:03,291 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/b6cca1209b0948e0bfa196185a1b9501 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/b6cca1209b0948e0bfa196185a1b9501 2024-11-28T01:42:03,292 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:03,292 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:03,292 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:03,292 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/03b3e84e9d9a4ab2a616be4394a86b2b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e991f00010b04117bd5fc5ab4311af59, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f1ad0d8246b84823aabb4d03adfcd1ea] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.6 K 2024-11-28T01:42:03,293 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b3e84e9d9a4ab2a616be4394a86b2b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732758121633 2024-11-28T01:42:03,293 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e991f00010b04117bd5fc5ab4311af59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732758122286 2024-11-28T01:42:03,294 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ad0d8246b84823aabb4d03adfcd1ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732758122973 2024-11-28T01:42:03,298 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into b6cca1209b0948e0bfa196185a1b9501(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:03,298 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:03,298 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=12, startTime=1732758122776; duration=0sec 2024-11-28T01:42:03,298 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:42:03,298 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:03,298 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:03,298 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:03,300 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:03,300 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:03,300 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:03,300 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9a90103e8330478a9f5118a5e0aebff3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a771652818e54a1c99698d008224a50d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5faaed00f39c4622aa9bb613fc04acbf] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.6 K 2024-11-28T01:42:03,301 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a90103e8330478a9f5118a5e0aebff3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732758121633 2024-11-28T01:42:03,301 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a771652818e54a1c99698d008224a50d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732758122286 2024-11-28T01:42:03,302 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5faaed00f39c4622aa9bb613fc04acbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732758122973 2024-11-28T01:42:03,313 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:03,314 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e04c92bac6ed42c2a7d3e940b7940d63 is 50, key is test_row_0/A:col10/1732758123089/Put/seqid=0 2024-11-28T01:42:03,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T01:42:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:03,319 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T01:42:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:03,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:03,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:03,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:03,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:03,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:03,321 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:03,322 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/90752d7d31974d7f86262c42bf55de39 is 50, key is test_row_0/B:col10/1732758123089/Put/seqid=0 2024-11-28T01:42:03,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742161_1337 (size=12983) 2024-11-28T01:42:03,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/f6fcff0cd3814f2fb80bc7f5fd102f9c is 50, key is test_row_0/A:col10/1732758123135/Put/seqid=0 2024-11-28T01:42:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742162_1338 (size=12983) 2024-11-28T01:42:03,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742163_1339 (size=12301) 2024-11-28T01:42:03,425 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/90752d7d31974d7f86262c42bf55de39 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/90752d7d31974d7f86262c42bf55de39 2024-11-28T01:42:03,431 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 90752d7d31974d7f86262c42bf55de39(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:03,431 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:03,431 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758123290; duration=0sec 2024-11-28T01:42:03,431 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:03,431 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:03,431 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T01:42:03,432 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T01:42:03,432 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T01:42:03,432 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. because compaction request was cancelled 2024-11-28T01:42:03,432 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:03,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T01:42:03,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758183488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758183495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758183495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758183497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758183603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758183612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758183613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758183613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,767 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e04c92bac6ed42c2a7d3e940b7940d63 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e04c92bac6ed42c2a7d3e940b7940d63 2024-11-28T01:42:03,775 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into e04c92bac6ed42c2a7d3e940b7940d63(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:03,775 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:03,775 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758123289; duration=0sec 2024-11-28T01:42:03,776 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:03,777 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T01:42:03,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758183812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,824 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/f6fcff0cd3814f2fb80bc7f5fd102f9c 2024-11-28T01:42:03,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758183823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758183823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758183823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:03,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9fcf8b4e69b447569d5190beef6378c4 is 50, key is test_row_0/B:col10/1732758123135/Put/seqid=0 2024-11-28T01:42:03,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742164_1340 (size=12301) 2024-11-28T01:42:03,880 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9fcf8b4e69b447569d5190beef6378c4 2024-11-28T01:42:03,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/800dfe508df64d5c933a97a75a9b307b is 50, key is test_row_0/C:col10/1732758123135/Put/seqid=0 2024-11-28T01:42:03,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742165_1341 (size=12301) 2024-11-28T01:42:03,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/800dfe508df64d5c933a97a75a9b307b 2024-11-28T01:42:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/f6fcff0cd3814f2fb80bc7f5fd102f9c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f6fcff0cd3814f2fb80bc7f5fd102f9c 2024-11-28T01:42:03,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f6fcff0cd3814f2fb80bc7f5fd102f9c, entries=150, sequenceid=424, filesize=12.0 K 2024-11-28T01:42:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9fcf8b4e69b447569d5190beef6378c4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9fcf8b4e69b447569d5190beef6378c4 2024-11-28T01:42:03,946 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9fcf8b4e69b447569d5190beef6378c4, entries=150, sequenceid=424, filesize=12.0 K 2024-11-28T01:42:03,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/800dfe508df64d5c933a97a75a9b307b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/800dfe508df64d5c933a97a75a9b307b 2024-11-28T01:42:03,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/800dfe508df64d5c933a97a75a9b307b, entries=150, sequenceid=424, filesize=12.0 K 2024-11-28T01:42:03,954 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 30f1052f6aa70fa647eb86fc190d7c15 in 635ms, sequenceid=424, compaction requested=true 2024-11-28T01:42:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:03,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-28T01:42:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-28T01:42:03,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-28T01:42:03,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 790 msec 2024-11-28T01:42:03,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 798 msec 2024-11-28T01:42:04,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:42:04,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:04,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:04,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:04,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:04,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:04,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:04,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/9e228f09403844589b75549dfbf79117 is 50, key is test_row_0/A:col10/1732758123491/Put/seqid=0 2024-11-28T01:42:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758184155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758184156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758184157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758184158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742166_1342 (size=14741) 2024-11-28T01:42:04,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/9e228f09403844589b75549dfbf79117 2024-11-28T01:42:04,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/36ee3532cc6940ef97505dba48370eca is 50, key is test_row_0/B:col10/1732758123491/Put/seqid=0 2024-11-28T01:42:04,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742167_1343 (size=12301) 2024-11-28T01:42:04,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758184273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758184273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758184273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758184273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T01:42:04,303 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:04,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-28T01:42:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T01:42:04,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758184303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,307 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:04,308 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 30f1052f6aa70fa647eb86fc190d7c15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:42:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,309 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:04,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T01:42:04,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T01:42:04,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:04,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758184483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758184483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758184483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758184483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T01:42:04,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:04,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T01:42:04,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/36ee3532cc6940ef97505dba48370eca 2024-11-28T01:42:04,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3fcd190c1ea74ca18e2ea3aee07db538 is 50, key is test_row_0/C:col10/1732758123491/Put/seqid=0 2024-11-28T01:42:04,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742168_1344 (size=12301) 2024-11-28T01:42:04,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T01:42:04,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:04,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758184792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758184792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758184792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758184793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:04,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T01:42:04,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:04,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:04,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T01:42:05,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3fcd190c1ea74ca18e2ea3aee07db538 2024-11-28T01:42:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/9e228f09403844589b75549dfbf79117 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9e228f09403844589b75549dfbf79117 2024-11-28T01:42:05,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9e228f09403844589b75549dfbf79117, entries=200, sequenceid=445, filesize=14.4 K 2024-11-28T01:42:05,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/36ee3532cc6940ef97505dba48370eca as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36ee3532cc6940ef97505dba48370eca 2024-11-28T01:42:05,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36ee3532cc6940ef97505dba48370eca, entries=150, sequenceid=445, filesize=12.0 K 2024-11-28T01:42:05,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3fcd190c1ea74ca18e2ea3aee07db538 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3fcd190c1ea74ca18e2ea3aee07db538 2024-11-28T01:42:05,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T01:42:05,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:05,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:05,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:05,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:05,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:05,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3fcd190c1ea74ca18e2ea3aee07db538, entries=150, sequenceid=445, filesize=12.0 K 2024-11-28T01:42:05,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 30f1052f6aa70fa647eb86fc190d7c15 in 957ms, sequenceid=445, compaction requested=true 2024-11-28T01:42:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:05,084 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:05,084 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:05,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:05,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:05,086 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:05,086 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:05,086 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/90752d7d31974d7f86262c42bf55de39, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9fcf8b4e69b447569d5190beef6378c4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36ee3532cc6940ef97505dba48370eca] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.7 K 2024-11-28T01:42:05,087 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:05,087 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:05,087 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:05,087 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e04c92bac6ed42c2a7d3e940b7940d63, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f6fcff0cd3814f2fb80bc7f5fd102f9c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9e228f09403844589b75549dfbf79117] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=39.1 K 2024-11-28T01:42:05,088 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e04c92bac6ed42c2a7d3e940b7940d63, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732758122973 2024-11-28T01:42:05,089 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 90752d7d31974d7f86262c42bf55de39, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732758122973 2024-11-28T01:42:05,090 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6fcff0cd3814f2fb80bc7f5fd102f9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732758123120 2024-11-28T01:42:05,090 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fcf8b4e69b447569d5190beef6378c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732758123120 2024-11-28T01:42:05,090 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e228f09403844589b75549dfbf79117, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732758123485 2024-11-28T01:42:05,091 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ee3532cc6940ef97505dba48370eca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732758123491 2024-11-28T01:42:05,109 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:05,110 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/53f7d6c516e84f8ebee3d615369f3b1b is 50, key is test_row_0/A:col10/1732758123491/Put/seqid=0 2024-11-28T01:42:05,114 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#293 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:05,115 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e0c43f6bab214d3ca7aad29b36acae82 is 50, key is test_row_0/B:col10/1732758123491/Put/seqid=0 2024-11-28T01:42:05,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742169_1345 (size=13085) 2024-11-28T01:42:05,159 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/53f7d6c516e84f8ebee3d615369f3b1b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/53f7d6c516e84f8ebee3d615369f3b1b 2024-11-28T01:42:05,165 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into 53f7d6c516e84f8ebee3d615369f3b1b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:05,165 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:05,165 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758125084; duration=0sec 2024-11-28T01:42:05,165 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:05,165 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:05,165 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:05,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:05,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742170_1346 (size=13085) 2024-11-28T01:42:05,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:05,168 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:05,168 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/b6cca1209b0948e0bfa196185a1b9501, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/836ddcbf43384bd1b4b9f7974182d877, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/800dfe508df64d5c933a97a75a9b307b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3fcd190c1ea74ca18e2ea3aee07db538] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=48.7 K 2024-11-28T01:42:05,169 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6cca1209b0948e0bfa196185a1b9501, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732758122286 2024-11-28T01:42:05,169 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 836ddcbf43384bd1b4b9f7974182d877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732758122973 2024-11-28T01:42:05,169 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 800dfe508df64d5c933a97a75a9b307b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732758123120 2024-11-28T01:42:05,170 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fcd190c1ea74ca18e2ea3aee07db538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732758123491 2024-11-28T01:42:05,190 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e0c43f6bab214d3ca7aad29b36acae82 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e0c43f6bab214d3ca7aad29b36acae82 2024-11-28T01:42:05,194 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:05,195 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/50d718fde54a4a31aa271996f648869b is 50, key is test_row_0/C:col10/1732758123491/Put/seqid=0 2024-11-28T01:42:05,195 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into e0c43f6bab214d3ca7aad29b36acae82(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:05,195 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:05,195 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758125084; duration=0sec 2024-11-28T01:42:05,195 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:05,195 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:05,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742171_1347 (size=13051) 2024-11-28T01:42:05,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T01:42:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:05,232 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:42:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:05,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/371e252a939549dfa59b196462c22985 is 50, key is test_row_0/A:col10/1732758124156/Put/seqid=0 2024-11-28T01:42:05,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742172_1348 (size=12301) 2024-11-28T01:42:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:05,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758185328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758185329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758185332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758185333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758185443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758185443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758185443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758185444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T01:42:05,607 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/50d718fde54a4a31aa271996f648869b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/50d718fde54a4a31aa271996f648869b 2024-11-28T01:42:05,614 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into 50d718fde54a4a31aa271996f648869b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:05,615 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:05,615 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=12, startTime=1732758125084; duration=0sec 2024-11-28T01:42:05,615 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:05,615 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:05,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/371e252a939549dfa59b196462c22985 2024-11-28T01:42:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758185652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758185652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758185653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758185653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/36dcaa6733ce405fa45feda86eb0c2c4 is 50, key is test_row_0/B:col10/1732758124156/Put/seqid=0 2024-11-28T01:42:05,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742173_1349 (size=12301) 2024-11-28T01:42:05,719 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/36dcaa6733ce405fa45feda86eb0c2c4 2024-11-28T01:42:05,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/42b53d9c7cd145b8b79cd27226dccd13 is 50, key is test_row_0/C:col10/1732758124156/Put/seqid=0 2024-11-28T01:42:05,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742174_1350 (size=12301) 2024-11-28T01:42:05,747 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/42b53d9c7cd145b8b79cd27226dccd13 2024-11-28T01:42:05,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/371e252a939549dfa59b196462c22985 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/371e252a939549dfa59b196462c22985 2024-11-28T01:42:05,761 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/371e252a939549dfa59b196462c22985, entries=150, sequenceid=460, filesize=12.0 K 2024-11-28T01:42:05,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/36dcaa6733ce405fa45feda86eb0c2c4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36dcaa6733ce405fa45feda86eb0c2c4 2024-11-28T01:42:05,769 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36dcaa6733ce405fa45feda86eb0c2c4, entries=150, sequenceid=460, filesize=12.0 K 2024-11-28T01:42:05,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/42b53d9c7cd145b8b79cd27226dccd13 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/42b53d9c7cd145b8b79cd27226dccd13 2024-11-28T01:42:05,776 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/42b53d9c7cd145b8b79cd27226dccd13, entries=150, sequenceid=460, filesize=12.0 K 2024-11-28T01:42:05,777 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 30f1052f6aa70fa647eb86fc190d7c15 in 545ms, sequenceid=460, compaction requested=false 2024-11-28T01:42:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-28T01:42:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-28T01:42:05,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-28T01:42:05,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4700 sec 2024-11-28T01:42:05,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4790 sec 2024-11-28T01:42:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:05,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T01:42:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:05,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/5e5d7be8b9634885907ae4f0cd7f92e0 is 50, key is test_row_0/A:col10/1732758125964/Put/seqid=0 2024-11-28T01:42:05,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742175_1351 (size=14741) 2024-11-28T01:42:05,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/5e5d7be8b9634885907ae4f0cd7f92e0 2024-11-28T01:42:05,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758185978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758185979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758185985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758185986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:05,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/973956b38c1f45448444a0db38b9cd40 is 50, key is test_row_0/B:col10/1732758125964/Put/seqid=0 2024-11-28T01:42:06,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742176_1352 (size=12301) 2024-11-28T01:42:06,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758186092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758186092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758186103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758186103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758186302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758186302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758186313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758186313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/973956b38c1f45448444a0db38b9cd40 2024-11-28T01:42:06,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/16ad831cf09449f0928b5155635961ff is 50, key is test_row_0/C:col10/1732758125964/Put/seqid=0 2024-11-28T01:42:06,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742177_1353 (size=12301) 2024-11-28T01:42:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T01:42:06,463 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:06,464 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-28T01:42:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T01:42:06,467 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:06,468 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:06,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T01:42:06,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758186612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758186612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T01:42:06,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758186623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758186623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:06,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T01:42:06,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:06,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:06,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T01:42:06,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/16ad831cf09449f0928b5155635961ff 2024-11-28T01:42:06,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/5e5d7be8b9634885907ae4f0cd7f92e0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/5e5d7be8b9634885907ae4f0cd7f92e0 2024-11-28T01:42:06,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/5e5d7be8b9634885907ae4f0cd7f92e0, entries=200, sequenceid=482, filesize=14.4 K 2024-11-28T01:42:06,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/973956b38c1f45448444a0db38b9cd40 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/973956b38c1f45448444a0db38b9cd40 2024-11-28T01:42:06,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/973956b38c1f45448444a0db38b9cd40, entries=150, sequenceid=482, filesize=12.0 K 2024-11-28T01:42:06,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/16ad831cf09449f0928b5155635961ff as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/16ad831cf09449f0928b5155635961ff 2024-11-28T01:42:06,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/16ad831cf09449f0928b5155635961ff, entries=150, sequenceid=482, filesize=12.0 K 2024-11-28T01:42:06,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 30f1052f6aa70fa647eb86fc190d7c15 in 887ms, sequenceid=482, compaction requested=true 2024-11-28T01:42:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:06,853 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:06,853 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:06,854 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:06,854 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:06,854 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,854 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/53f7d6c516e84f8ebee3d615369f3b1b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/371e252a939549dfa59b196462c22985, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/5e5d7be8b9634885907ae4f0cd7f92e0] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=39.2 K 2024-11-28T01:42:06,855 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:06,855 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:06,855 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,855 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e0c43f6bab214d3ca7aad29b36acae82, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36dcaa6733ce405fa45feda86eb0c2c4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/973956b38c1f45448444a0db38b9cd40] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.8 K 2024-11-28T01:42:06,855 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53f7d6c516e84f8ebee3d615369f3b1b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732758123491 2024-11-28T01:42:06,855 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e0c43f6bab214d3ca7aad29b36acae82, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732758123491 2024-11-28T01:42:06,855 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 371e252a939549dfa59b196462c22985, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732758124145 2024-11-28T01:42:06,856 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 36dcaa6733ce405fa45feda86eb0c2c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732758124145 2024-11-28T01:42:06,856 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e5d7be8b9634885907ae4f0cd7f92e0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758125322 2024-11-28T01:42:06,856 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 973956b38c1f45448444a0db38b9cd40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758125322 2024-11-28T01:42:06,871 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:06,872 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/071fee6de9aa4fa49ca002f455933666 is 50, key is test_row_0/B:col10/1732758125964/Put/seqid=0 2024-11-28T01:42:06,890 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:06,891 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/a887c3ea54c34b4b95d537bdcdcc43dd is 50, key is test_row_0/A:col10/1732758125964/Put/seqid=0 2024-11-28T01:42:06,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742178_1354 (size=13187) 2024-11-28T01:42:06,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742179_1355 (size=13187) 2024-11-28T01:42:06,915 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/a887c3ea54c34b4b95d537bdcdcc43dd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a887c3ea54c34b4b95d537bdcdcc43dd 2024-11-28T01:42:06,921 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into a887c3ea54c34b4b95d537bdcdcc43dd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:06,921 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:06,921 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758126853; duration=0sec 2024-11-28T01:42:06,921 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:06,921 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:06,921 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:06,923 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:06,923 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:06,923 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,923 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/50d718fde54a4a31aa271996f648869b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/42b53d9c7cd145b8b79cd27226dccd13, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/16ad831cf09449f0928b5155635961ff] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.8 K 2024-11-28T01:42:06,924 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50d718fde54a4a31aa271996f648869b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732758123491 2024-11-28T01:42:06,924 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b53d9c7cd145b8b79cd27226dccd13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732758124145 2024-11-28T01:42:06,925 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ad831cf09449f0928b5155635961ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758125322 2024-11-28T01:42:06,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T01:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:06,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:06,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/789ccf78649d41a5b117ed23b7a17d6e is 50, key is test_row_0/A:col10/1732758125983/Put/seqid=0 2024-11-28T01:42:06,950 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:06,951 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/9be84ee0a09947618ac241339f91b159 is 50, key is test_row_0/C:col10/1732758125964/Put/seqid=0 2024-11-28T01:42:06,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742181_1357 (size=13153) 2024-11-28T01:42:06,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742180_1356 (size=12301) 2024-11-28T01:42:06,977 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/9be84ee0a09947618ac241339f91b159 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9be84ee0a09947618ac241339f91b159 2024-11-28T01:42:06,983 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into 9be84ee0a09947618ac241339f91b159(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:06,983 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:06,983 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=13, startTime=1732758126853; duration=0sec 2024-11-28T01:42:06,984 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:06,984 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T01:42:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:07,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:07,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758127138, max timestamp=1732758127166Num uniques 859; 2024-11-28T01:42:07,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758187165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:42:07,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758127138, max timestamp=1732758127166Num uniques 859; 2024-11-28T01:42:07,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:42:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758127138, max timestamp=1732758127166Num uniques 859; 2024-11-28T01:42:07,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:42:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758187210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758187211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758187213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758187283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,304 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/071fee6de9aa4fa49ca002f455933666 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/071fee6de9aa4fa49ca002f455933666 2024-11-28T01:42:07,310 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 071fee6de9aa4fa49ca002f455933666(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:07,310 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:07,310 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758126853; duration=0sec 2024-11-28T01:42:07,310 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:07,310 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:07,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758187323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758187323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758187323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,370 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/789ccf78649d41a5b117ed23b7a17d6e 2024-11-28T01:42:07,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/eae3da68cf4d45d3916669b4a542ea2f is 50, key is test_row_0/B:col10/1732758125983/Put/seqid=0 2024-11-28T01:42:07,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742182_1358 (size=12301) 2024-11-28T01:42:07,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758187492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758187533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758187534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758187534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T01:42:07,794 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/eae3da68cf4d45d3916669b4a542ea2f 2024-11-28T01:42:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758187802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/2939923b3c074689adb46f654fba66b4 is 50, key is test_row_0/C:col10/1732758125983/Put/seqid=0 2024-11-28T01:42:07,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742183_1359 (size=12301) 2024-11-28T01:42:07,828 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/2939923b3c074689adb46f654fba66b4 2024-11-28T01:42:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/789ccf78649d41a5b117ed23b7a17d6e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/789ccf78649d41a5b117ed23b7a17d6e 2024-11-28T01:42:07,846 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/789ccf78649d41a5b117ed23b7a17d6e, entries=150, sequenceid=495, filesize=12.0 K 2024-11-28T01:42:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758187842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758187842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758187843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:07,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/eae3da68cf4d45d3916669b4a542ea2f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/eae3da68cf4d45d3916669b4a542ea2f 2024-11-28T01:42:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:07,861 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/eae3da68cf4d45d3916669b4a542ea2f, entries=150, sequenceid=495, filesize=12.0 K 2024-11-28T01:42:07,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/2939923b3c074689adb46f654fba66b4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/2939923b3c074689adb46f654fba66b4 2024-11-28T01:42:07,874 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/2939923b3c074689adb46f654fba66b4, entries=150, sequenceid=495, filesize=12.0 K 2024-11-28T01:42:07,878 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 30f1052f6aa70fa647eb86fc190d7c15 in 949ms, sequenceid=495, compaction requested=false 2024-11-28T01:42:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:07,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-28T01:42:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-28T01:42:07,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-28T01:42:07,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-28T01:42:07,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4190 sec 2024-11-28T01:42:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:08,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T01:42:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:08,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/2af743b2194c4586a1d1408a6f57be96 is 50, key is test_row_0/A:col10/1732758127190/Put/seqid=0 2024-11-28T01:42:08,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742184_1360 (size=14741) 2024-11-28T01:42:08,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/2af743b2194c4586a1d1408a6f57be96 2024-11-28T01:42:08,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4 is 50, key is test_row_0/B:col10/1732758127190/Put/seqid=0 2024-11-28T01:42:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758188357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758188358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758188360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758188366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758188367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742185_1361 (size=12301) 2024-11-28T01:42:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758188473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758188473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758188473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T01:42:08,613 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:08,614 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-28T01:42:08,617 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:08,618 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:08,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T01:42:08,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758188683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758188683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758188692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T01:42:08,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:08,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:08,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:08,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:08,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:08,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:08,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4 2024-11-28T01:42:08,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/32dd22c1a1574ff98c61d388059a6054 is 50, key is test_row_0/C:col10/1732758127190/Put/seqid=0 2024-11-28T01:42:08,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742186_1362 (size=12301) 2024-11-28T01:42:08,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:08,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:08,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T01:42:08,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758188993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758188993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758189002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:09,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:09,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:09,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/32dd22c1a1574ff98c61d388059a6054 2024-11-28T01:42:09,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/2af743b2194c4586a1d1408a6f57be96 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/2af743b2194c4586a1d1408a6f57be96 2024-11-28T01:42:09,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/2af743b2194c4586a1d1408a6f57be96, entries=200, sequenceid=532, filesize=14.4 K 2024-11-28T01:42:09,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4 2024-11-28T01:42:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T01:42:09,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4, entries=150, sequenceid=532, filesize=12.0 K 2024-11-28T01:42:09,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/32dd22c1a1574ff98c61d388059a6054 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/32dd22c1a1574ff98c61d388059a6054 2024-11-28T01:42:09,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/32dd22c1a1574ff98c61d388059a6054, entries=150, sequenceid=532, filesize=12.0 K 2024-11-28T01:42:09,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 30f1052f6aa70fa647eb86fc190d7c15 in 945ms, sequenceid=532, compaction requested=true 2024-11-28T01:42:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:09,261 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:09,261 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:09,263 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:09,263 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:09,263 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,263 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a887c3ea54c34b4b95d537bdcdcc43dd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/789ccf78649d41a5b117ed23b7a17d6e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/2af743b2194c4586a1d1408a6f57be96] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=39.3 K 2024-11-28T01:42:09,263 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:09,263 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:09,264 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,264 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/071fee6de9aa4fa49ca002f455933666, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/eae3da68cf4d45d3916669b4a542ea2f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.9 K 2024-11-28T01:42:09,264 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a887c3ea54c34b4b95d537bdcdcc43dd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758125322 2024-11-28T01:42:09,264 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 071fee6de9aa4fa49ca002f455933666, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758125322 2024-11-28T01:42:09,264 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 789ccf78649d41a5b117ed23b7a17d6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732758125968 2024-11-28T01:42:09,265 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting eae3da68cf4d45d3916669b4a542ea2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732758125968 2024-11-28T01:42:09,265 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2af743b2194c4586a1d1408a6f57be96, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732758127190 2024-11-28T01:42:09,265 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a4bb2b5f00943e3bcbab4d2e2cca6d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732758127190 2024-11-28T01:42:09,274 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:09,275 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/040723eb4b3448be86f8e02fc0551353 is 50, key is test_row_0/B:col10/1732758127190/Put/seqid=0 2024-11-28T01:42:09,275 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:09,277 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1b6c381d5a25492f86408979d9be05b1 is 50, key is test_row_0/A:col10/1732758127190/Put/seqid=0 2024-11-28T01:42:09,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742187_1363 (size=13289) 2024-11-28T01:42:09,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742188_1364 (size=13289) 2024-11-28T01:42:09,299 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/040723eb4b3448be86f8e02fc0551353 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/040723eb4b3448be86f8e02fc0551353 2024-11-28T01:42:09,305 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 040723eb4b3448be86f8e02fc0551353(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:09,306 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:09,306 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758129261; duration=0sec 2024-11-28T01:42:09,306 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:09,306 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:09,306 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:09,309 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:09,309 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:09,310 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,310 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9be84ee0a09947618ac241339f91b159, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/2939923b3c074689adb46f654fba66b4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/32dd22c1a1574ff98c61d388059a6054] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=36.9 K 2024-11-28T01:42:09,310 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be84ee0a09947618ac241339f91b159, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732758125322 2024-11-28T01:42:09,311 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2939923b3c074689adb46f654fba66b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732758125968 2024-11-28T01:42:09,311 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 32dd22c1a1574ff98c61d388059a6054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732758127190 2024-11-28T01:42:09,313 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1b6c381d5a25492f86408979d9be05b1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1b6c381d5a25492f86408979d9be05b1 2024-11-28T01:42:09,319 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into 1b6c381d5a25492f86408979d9be05b1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:09,319 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:09,319 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758129261; duration=0sec 2024-11-28T01:42:09,319 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:09,319 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:09,321 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:09,322 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/ed8559b90afa4acd9366160bc8b0ed39 is 50, key is test_row_0/C:col10/1732758127190/Put/seqid=0 2024-11-28T01:42:09,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742189_1365 (size=13255) 2024-11-28T01:42:09,339 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/ed8559b90afa4acd9366160bc8b0ed39 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ed8559b90afa4acd9366160bc8b0ed39 2024-11-28T01:42:09,346 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into ed8559b90afa4acd9366160bc8b0ed39(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:09,346 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:09,346 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=13, startTime=1732758129261; duration=0sec 2024-11-28T01:42:09,346 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:09,346 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:09,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:42:09,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:09,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:09,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:09,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:09,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:09,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:09,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/08017665ba484dcea75394c34a13c2a1 is 50, key is test_row_0/A:col10/1732758129373/Put/seqid=0 2024-11-28T01:42:09,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742190_1366 (size=12301) 2024-11-28T01:42:09,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758189413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758189416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758189502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758189512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758189512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758189522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758189522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:09,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:09,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:09,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:09,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758189732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758189732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T01:42:09,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/08017665ba484dcea75394c34a13c2a1 2024-11-28T01:42:09,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/fb054997e1144b668c104fb41de0dbc2 is 50, key is test_row_0/B:col10/1732758129373/Put/seqid=0 2024-11-28T01:42:09,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742191_1367 (size=12301) 2024-11-28T01:42:09,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:09,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:10,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:10,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:10,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758190042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758190042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:10,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:10,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:10,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:10,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:10,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:10,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/fb054997e1144b668c104fb41de0dbc2 2024-11-28T01:42:10,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/d72908d6f4ed45c6a23718c1864110c3 is 50, key is test_row_0/C:col10/1732758129373/Put/seqid=0 2024-11-28T01:42:10,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742192_1368 (size=12301) 2024-11-28T01:42:10,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/d72908d6f4ed45c6a23718c1864110c3 2024-11-28T01:42:10,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/08017665ba484dcea75394c34a13c2a1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/08017665ba484dcea75394c34a13c2a1 2024-11-28T01:42:10,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/08017665ba484dcea75394c34a13c2a1, entries=150, sequenceid=548, filesize=12.0 K 2024-11-28T01:42:10,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/fb054997e1144b668c104fb41de0dbc2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fb054997e1144b668c104fb41de0dbc2 2024-11-28T01:42:10,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fb054997e1144b668c104fb41de0dbc2, entries=150, sequenceid=548, filesize=12.0 K 2024-11-28T01:42:10,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/d72908d6f4ed45c6a23718c1864110c3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d72908d6f4ed45c6a23718c1864110c3 2024-11-28T01:42:10,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d72908d6f4ed45c6a23718c1864110c3, entries=150, sequenceid=548, filesize=12.0 K 2024-11-28T01:42:10,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 30f1052f6aa70fa647eb86fc190d7c15 in 879ms, sequenceid=548, compaction requested=false 2024-11-28T01:42:10,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:10,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T01:42:10,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:10,310 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:42:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:10,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e72c59c7eef64e4d8826de28521f6d3f is 50, key is test_row_0/A:col10/1732758129414/Put/seqid=0 2024-11-28T01:42:10,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742193_1369 (size=12301) 2024-11-28T01:42:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:10,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:10,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758190535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758190537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758190538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758190552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758190552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758190653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758190653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758190653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e72c59c7eef64e4d8826de28521f6d3f 2024-11-28T01:42:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9e14e92d91544f4cb42042d29b0f13cb is 50, key is test_row_0/B:col10/1732758129414/Put/seqid=0 2024-11-28T01:42:10,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742194_1370 (size=12301) 2024-11-28T01:42:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T01:42:10,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758190863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758190863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:10,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758190864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:10,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,134 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9e14e92d91544f4cb42042d29b0f13cb 2024-11-28T01:42:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/e8a84f185f0c4189a7a167bd7b047d48 is 50, key is test_row_0/C:col10/1732758129414/Put/seqid=0 2024-11-28T01:42:11,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742195_1371 (size=12301) 2024-11-28T01:42:11,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758191173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758191172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758191175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,546 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=567 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/e8a84f185f0c4189a7a167bd7b047d48 2024-11-28T01:42:11,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/e72c59c7eef64e4d8826de28521f6d3f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e72c59c7eef64e4d8826de28521f6d3f 2024-11-28T01:42:11,556 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e72c59c7eef64e4d8826de28521f6d3f, entries=150, sequenceid=567, filesize=12.0 K 2024-11-28T01:42:11,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/9e14e92d91544f4cb42042d29b0f13cb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9e14e92d91544f4cb42042d29b0f13cb 2024-11-28T01:42:11,561 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9e14e92d91544f4cb42042d29b0f13cb, entries=150, sequenceid=567, filesize=12.0 K 2024-11-28T01:42:11,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/e8a84f185f0c4189a7a167bd7b047d48 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/e8a84f185f0c4189a7a167bd7b047d48 2024-11-28T01:42:11,565 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/e8a84f185f0c4189a7a167bd7b047d48, entries=150, sequenceid=567, filesize=12.0 K 2024-11-28T01:42:11,566 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 30f1052f6aa70fa647eb86fc190d7c15 in 1256ms, sequenceid=567, compaction requested=true 2024-11-28T01:42:11,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:11,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:11,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-28T01:42:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-28T01:42:11,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-28T01:42:11,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9500 sec 2024-11-28T01:42:11,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.9570 sec 2024-11-28T01:42:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:11,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:42:11,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:11,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:11,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:11,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:11,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:11,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:11,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/7897fdffb57b42739e991105ba5cc4f7 is 50, key is test_row_0/A:col10/1732758130537/Put/seqid=0 2024-11-28T01:42:11,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742196_1372 (size=14741) 2024-11-28T01:42:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758191636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758191637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758191682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758191682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758191693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758191752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758191752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758191962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758191962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:11,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/7897fdffb57b42739e991105ba5cc4f7 2024-11-28T01:42:11,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/792a325edc96410dacfc244c35712080 is 50, key is test_row_0/B:col10/1732758130537/Put/seqid=0 2024-11-28T01:42:11,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742197_1373 (size=12301) 2024-11-28T01:42:12,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/792a325edc96410dacfc244c35712080 2024-11-28T01:42:12,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/f433507d972f4e86aa86561c1810e042 is 50, key is test_row_0/C:col10/1732758130537/Put/seqid=0 2024-11-28T01:42:12,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742198_1374 (size=12301) 2024-11-28T01:42:12,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/f433507d972f4e86aa86561c1810e042 2024-11-28T01:42:12,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/7897fdffb57b42739e991105ba5cc4f7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7897fdffb57b42739e991105ba5cc4f7 2024-11-28T01:42:12,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7897fdffb57b42739e991105ba5cc4f7, entries=200, sequenceid=581, filesize=14.4 K 2024-11-28T01:42:12,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/792a325edc96410dacfc244c35712080 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/792a325edc96410dacfc244c35712080 2024-11-28T01:42:12,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/792a325edc96410dacfc244c35712080, entries=150, sequenceid=581, filesize=12.0 K 2024-11-28T01:42:12,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/f433507d972f4e86aa86561c1810e042 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/f433507d972f4e86aa86561c1810e042 2024-11-28T01:42:12,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/f433507d972f4e86aa86561c1810e042, entries=150, sequenceid=581, filesize=12.0 K 2024-11-28T01:42:12,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 30f1052f6aa70fa647eb86fc190d7c15 in 470ms, sequenceid=581, compaction requested=true 2024-11-28T01:42:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:12,045 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:12,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:12,045 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:12,047 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:12,047 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:12,047 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:12,047 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:12,047 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:12,047 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:12,047 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/040723eb4b3448be86f8e02fc0551353, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fb054997e1144b668c104fb41de0dbc2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9e14e92d91544f4cb42042d29b0f13cb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/792a325edc96410dacfc244c35712080] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=49.0 K 2024-11-28T01:42:12,047 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1b6c381d5a25492f86408979d9be05b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/08017665ba484dcea75394c34a13c2a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e72c59c7eef64e4d8826de28521f6d3f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7897fdffb57b42739e991105ba5cc4f7] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=51.4 K 2024-11-28T01:42:12,047 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 040723eb4b3448be86f8e02fc0551353, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732758127190 2024-11-28T01:42:12,048 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b6c381d5a25492f86408979d9be05b1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732758127190 2024-11-28T01:42:12,048 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08017665ba484dcea75394c34a13c2a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732758128356 2024-11-28T01:42:12,048 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e72c59c7eef64e4d8826de28521f6d3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732758129395 2024-11-28T01:42:12,048 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7897fdffb57b42739e991105ba5cc4f7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732758130522 2024-11-28T01:42:12,049 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting fb054997e1144b668c104fb41de0dbc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732758128356 2024-11-28T01:42:12,049 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e14e92d91544f4cb42042d29b0f13cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732758129395 2024-11-28T01:42:12,049 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 792a325edc96410dacfc244c35712080, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732758130522 2024-11-28T01:42:12,058 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:12,058 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/9f677725e59f42eaa9fe4373f08bd894 is 50, key is test_row_0/A:col10/1732758130537/Put/seqid=0 2024-11-28T01:42:12,060 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:12,061 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/1cb85e79a6c849129ddfe9ab700a77d2 is 50, key is test_row_0/B:col10/1732758130537/Put/seqid=0 2024-11-28T01:42:12,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742200_1376 (size=13425) 2024-11-28T01:42:12,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742199_1375 (size=13425) 2024-11-28T01:42:12,086 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/9f677725e59f42eaa9fe4373f08bd894 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9f677725e59f42eaa9fe4373f08bd894 2024-11-28T01:42:12,092 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into 9f677725e59f42eaa9fe4373f08bd894(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:12,092 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:12,092 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=12, startTime=1732758132045; duration=0sec 2024-11-28T01:42:12,092 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:12,092 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:12,092 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:12,094 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:12,094 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:12,094 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:12,094 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ed8559b90afa4acd9366160bc8b0ed39, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d72908d6f4ed45c6a23718c1864110c3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/e8a84f185f0c4189a7a167bd7b047d48, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/f433507d972f4e86aa86561c1810e042] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=49.0 K 2024-11-28T01:42:12,096 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed8559b90afa4acd9366160bc8b0ed39, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1732758127190 2024-11-28T01:42:12,096 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d72908d6f4ed45c6a23718c1864110c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732758128356 2024-11-28T01:42:12,096 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a84f185f0c4189a7a167bd7b047d48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=567, earliestPutTs=1732758129395 2024-11-28T01:42:12,097 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f433507d972f4e86aa86561c1810e042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732758130522 2024-11-28T01:42:12,107 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:12,108 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c10994d9cce74f22b90b182c2f805861 is 50, key is test_row_0/C:col10/1732758130537/Put/seqid=0 2024-11-28T01:42:12,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742201_1377 (size=13391) 2024-11-28T01:42:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:12,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:42:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:12,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:12,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/95dbbfffce95451a9354cfe27577adf4 is 50, key is test_row_0/A:col10/1732758131592/Put/seqid=0 2024-11-28T01:42:12,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742202_1378 (size=14741) 2024-11-28T01:42:12,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/95dbbfffce95451a9354cfe27577adf4 2024-11-28T01:42:12,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0c540b569d7a4fc6b543998982eb08ac is 50, key is test_row_0/B:col10/1732758131592/Put/seqid=0 2024-11-28T01:42:12,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742203_1379 (size=12301) 2024-11-28T01:42:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758192304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758192305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758192412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758192413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,475 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/1cb85e79a6c849129ddfe9ab700a77d2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1cb85e79a6c849129ddfe9ab700a77d2 2024-11-28T01:42:12,480 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 1cb85e79a6c849129ddfe9ab700a77d2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:12,480 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:12,480 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=12, startTime=1732758132045; duration=0sec 2024-11-28T01:42:12,480 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:12,480 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:12,520 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/c10994d9cce74f22b90b182c2f805861 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c10994d9cce74f22b90b182c2f805861 2024-11-28T01:42:12,524 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into c10994d9cce74f22b90b182c2f805861(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:12,524 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:12,524 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=12, startTime=1732758132045; duration=0sec 2024-11-28T01:42:12,524 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:12,524 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35592 deadline: 1732758192622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35606 deadline: 1732758192622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35648 deadline: 1732758192692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,694 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 30f1052f6aa70fa647eb86fc190d7c15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:42:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0c540b569d7a4fc6b543998982eb08ac 2024-11-28T01:42:12,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35582 deadline: 1732758192702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35584 deadline: 1732758192703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:12,705 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 30f1052f6aa70fa647eb86fc190d7c15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:42:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,706 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 30f1052f6aa70fa647eb86fc190d7c15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:42:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:12,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:12,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/4c12ba54dcbb4ac3a03799e84fc6ab3c is 50, key is test_row_0/C:col10/1732758131592/Put/seqid=0 2024-11-28T01:42:12,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742204_1380 (size=12301) 2024-11-28T01:42:12,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=602 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/4c12ba54dcbb4ac3a03799e84fc6ab3c 2024-11-28T01:42:12,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/95dbbfffce95451a9354cfe27577adf4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/95dbbfffce95451a9354cfe27577adf4 2024-11-28T01:42:12,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/95dbbfffce95451a9354cfe27577adf4, entries=200, sequenceid=602, filesize=14.4 K 2024-11-28T01:42:12,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0c540b569d7a4fc6b543998982eb08ac as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0c540b569d7a4fc6b543998982eb08ac 2024-11-28T01:42:12,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0c540b569d7a4fc6b543998982eb08ac, entries=150, sequenceid=602, filesize=12.0 K 2024-11-28T01:42:12,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/4c12ba54dcbb4ac3a03799e84fc6ab3c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c12ba54dcbb4ac3a03799e84fc6ab3c 2024-11-28T01:42:12,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c12ba54dcbb4ac3a03799e84fc6ab3c, entries=150, sequenceid=602, filesize=12.0 K 2024-11-28T01:42:12,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 30f1052f6aa70fa647eb86fc190d7c15 in 467ms, sequenceid=602, compaction requested=false 2024-11-28T01:42:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T01:42:12,792 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:12,793 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-28T01:42:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T01:42:12,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:12,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:12,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T01:42:12,927 INFO [Thread-1272 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1272. 2024-11-28T01:42:12,927 DEBUG [Thread-1272 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:12,927 DEBUG [Thread-1272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,927 INFO [Thread-1278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1278. 2024-11-28T01:42:12,927 DEBUG [Thread-1278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:12,927 DEBUG [Thread-1278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:12,927 DEBUG [Thread-1278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:12,927 INFO [Thread-1280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1280. 2024-11-28T01:42:12,927 DEBUG [Thread-1280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:12,927 DEBUG [Thread-1280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,928 DEBUG [Thread-1280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:12,928 INFO [Thread-1274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1274. 2024-11-28T01:42:12,928 DEBUG [Thread-1274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:12,928 DEBUG [Thread-1274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,928 DEBUG [Thread-1274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:12,929 INFO [Thread-1276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1276. 2024-11-28T01:42:12,929 DEBUG [Thread-1276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:12,930 DEBUG [Thread-1276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,930 DEBUG [Thread-1276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:12,930 DEBUG [Thread-1272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:12,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:42:12,935 INFO [Thread-1261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1261. 2024-11-28T01:42:12,935 INFO [Thread-1269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1269. 2024-11-28T01:42:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:12,935 DEBUG [Thread-1261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:12,935 DEBUG [Thread-1269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:12,935 DEBUG [Thread-1261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,935 DEBUG [Thread-1269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:12,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:12,935 DEBUG [Thread-1261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,935 DEBUG [Thread-1269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:12,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:12,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:12,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1fa4a5e633ae4f35bbae270624fece2f is 50, key is test_row_0/A:col10/1732758132933/Put/seqid=0 2024-11-28T01:42:12,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742205_1381 (size=12301) 2024-11-28T01:42:12,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:12,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:12,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:12,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:12,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:12,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T01:42:13,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1fa4a5e633ae4f35bbae270624fece2f 2024-11-28T01:42:13,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e2935e414a28448cae79c03df981d242 is 50, key is test_row_0/B:col10/1732758132933/Put/seqid=0 2024-11-28T01:42:13,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742206_1382 (size=12301) 2024-11-28T01:42:13,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T01:42:13,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e2935e414a28448cae79c03df981d242 2024-11-28T01:42:13,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/255f772c22134630986771f1fd0a7754 is 50, key is test_row_0/C:col10/1732758132933/Put/seqid=0 2024-11-28T01:42:13,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742207_1383 (size=12301) 2024-11-28T01:42:13,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:13,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T01:42:14,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:14,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:14,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/255f772c22134630986771f1fd0a7754 2024-11-28T01:42:14,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/1fa4a5e633ae4f35bbae270624fece2f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1fa4a5e633ae4f35bbae270624fece2f 2024-11-28T01:42:14,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1fa4a5e633ae4f35bbae270624fece2f, entries=150, sequenceid=618, filesize=12.0 K 2024-11-28T01:42:14,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/e2935e414a28448cae79c03df981d242 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e2935e414a28448cae79c03df981d242 2024-11-28T01:42:14,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e2935e414a28448cae79c03df981d242, entries=150, sequenceid=618, filesize=12.0 K 2024-11-28T01:42:14,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/255f772c22134630986771f1fd0a7754 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/255f772c22134630986771f1fd0a7754 2024-11-28T01:42:14,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. as already flushing 2024-11-28T01:42:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:14,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/255f772c22134630986771f1fd0a7754, entries=150, sequenceid=618, filesize=12.0 K 2024-11-28T01:42:14,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 30f1052f6aa70fa647eb86fc190d7c15 in 1243ms, sequenceid=618, compaction requested=true 2024-11-28T01:42:14,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:14,178 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30f1052f6aa70fa647eb86fc190d7c15:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:14,178 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:14,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:14,178 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:14,178 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:14,178 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/A is initiating minor compaction (all files) 2024-11-28T01:42:14,178 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/B is initiating minor compaction (all files) 2024-11-28T01:42:14,178 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/A in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,179 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/B in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,179 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9f677725e59f42eaa9fe4373f08bd894, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/95dbbfffce95451a9354cfe27577adf4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1fa4a5e633ae4f35bbae270624fece2f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=39.5 K 2024-11-28T01:42:14,179 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1cb85e79a6c849129ddfe9ab700a77d2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0c540b569d7a4fc6b543998982eb08ac, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e2935e414a28448cae79c03df981d242] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=37.1 K 2024-11-28T01:42:14,179 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f677725e59f42eaa9fe4373f08bd894, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732758130522 2024-11-28T01:42:14,179 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb85e79a6c849129ddfe9ab700a77d2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732758130522 2024-11-28T01:42:14,179 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95dbbfffce95451a9354cfe27577adf4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=602, earliestPutTs=1732758131592 2024-11-28T01:42:14,179 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c540b569d7a4fc6b543998982eb08ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=602, earliestPutTs=1732758131592 2024-11-28T01:42:14,179 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa4a5e633ae4f35bbae270624fece2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1732758132290 2024-11-28T01:42:14,179 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e2935e414a28448cae79c03df981d242, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1732758132290 2024-11-28T01:42:14,185 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#A#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:14,186 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/daca8ed12491411ab2ba3bde37f70f20 is 50, key is test_row_0/A:col10/1732758132933/Put/seqid=0 2024-11-28T01:42:14,186 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:14,187 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0042d7eb11d342cf879b8dbe99693648 is 50, key is test_row_0/B:col10/1732758132933/Put/seqid=0 2024-11-28T01:42:14,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742208_1384 (size=13527) 2024-11-28T01:42:14,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742209_1385 (size=13527) 2024-11-28T01:42:14,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T01:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-28T01:42:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-28T01:42:14,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-28T01:42:14,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5320 sec 2024-11-28T01:42:14,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.5370 sec 2024-11-28T01:42:14,596 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/daca8ed12491411ab2ba3bde37f70f20 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/daca8ed12491411ab2ba3bde37f70f20 2024-11-28T01:42:14,600 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/0042d7eb11d342cf879b8dbe99693648 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0042d7eb11d342cf879b8dbe99693648 2024-11-28T01:42:14,601 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/A of 30f1052f6aa70fa647eb86fc190d7c15 into daca8ed12491411ab2ba3bde37f70f20(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:14,601 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:14,601 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/A, priority=13, startTime=1732758134177; duration=0sec 2024-11-28T01:42:14,601 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:14,601 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:A 2024-11-28T01:42:14,601 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:14,602 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:14,602 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 30f1052f6aa70fa647eb86fc190d7c15/C is initiating minor compaction (all files) 2024-11-28T01:42:14,602 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30f1052f6aa70fa647eb86fc190d7c15/C in TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:14,602 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c10994d9cce74f22b90b182c2f805861, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c12ba54dcbb4ac3a03799e84fc6ab3c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/255f772c22134630986771f1fd0a7754] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp, totalSize=37.1 K 2024-11-28T01:42:14,602 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10994d9cce74f22b90b182c2f805861, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1732758130522 2024-11-28T01:42:14,603 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c12ba54dcbb4ac3a03799e84fc6ab3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=602, earliestPutTs=1732758131592 2024-11-28T01:42:14,603 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 255f772c22134630986771f1fd0a7754, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1732758132290 2024-11-28T01:42:14,605 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/B of 30f1052f6aa70fa647eb86fc190d7c15 into 0042d7eb11d342cf879b8dbe99693648(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:14,605 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:14,605 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/B, priority=13, startTime=1732758134178; duration=0sec 2024-11-28T01:42:14,605 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:14,605 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:B 2024-11-28T01:42:14,609 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30f1052f6aa70fa647eb86fc190d7c15#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:14,609 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3a2ec7c5ecb74c8cacbf3a8ba11e7dfb is 50, key is test_row_0/C:col10/1732758132933/Put/seqid=0 2024-11-28T01:42:14,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742210_1386 (size=13493) 2024-11-28T01:42:14,617 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3a2ec7c5ecb74c8cacbf3a8ba11e7dfb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3a2ec7c5ecb74c8cacbf3a8ba11e7dfb 2024-11-28T01:42:14,621 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30f1052f6aa70fa647eb86fc190d7c15/C of 30f1052f6aa70fa647eb86fc190d7c15 into 3a2ec7c5ecb74c8cacbf3a8ba11e7dfb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:14,621 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30f1052f6aa70fa647eb86fc190d7c15: 2024-11-28T01:42:14,621 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., storeName=30f1052f6aa70fa647eb86fc190d7c15/C, priority=13, startTime=1732758134178; duration=0sec 2024-11-28T01:42:14,621 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:14,621 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30f1052f6aa70fa647eb86fc190d7c15:C 2024-11-28T01:42:14,704 INFO [Thread-1265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1265. 2024-11-28T01:42:14,704 DEBUG [Thread-1265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:14,704 DEBUG [Thread-1265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,704 DEBUG [Thread-1265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:14,724 INFO [Thread-1263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1263. 2024-11-28T01:42:14,724 INFO [Thread-1267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1267. 2024-11-28T01:42:14,724 DEBUG [Thread-1267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:14,724 DEBUG [Thread-1263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:14,724 DEBUG [Thread-1267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,724 DEBUG [Thread-1263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,724 DEBUG [Thread-1267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,724 DEBUG [Thread-1263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:14,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T01:42:14,953 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2611 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2483 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2572 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2624 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2480 2024-11-28T01:42:14,953 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T01:42:14,953 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:42:14,954 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:42:14,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:14,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:14,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T01:42:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T01:42:14,957 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758134957"}]},"ts":"1732758134957"} 2024-11-28T01:42:14,959 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T01:42:14,959 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T01:42:14,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:42:14,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, UNASSIGN}] 2024-11-28T01:42:14,961 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, UNASSIGN 2024-11-28T01:42:14,962 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=30f1052f6aa70fa647eb86fc190d7c15, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:14,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, UNASSIGN because future has completed 2024-11-28T01:42:14,963 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:42:14,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:42:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T01:42:15,115 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:15,115 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 30f1052f6aa70fa647eb86fc190d7c15, disabling compactions & flushes 2024-11-28T01:42:15,116 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. after waiting 0 ms 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:15,116 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 30f1052f6aa70fa647eb86fc190d7c15 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=A 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=B 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 30f1052f6aa70fa647eb86fc190d7c15, store=C 2024-11-28T01:42:15,116 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:15,120 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/6c4318cd42804ded81ffa5a73f1a1e4f is 50, key is test_row_2/A:col10/1732758134722/Put/seqid=0 2024-11-28T01:42:15,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742211_1387 (size=7415) 2024-11-28T01:42:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T01:42:15,524 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/6c4318cd42804ded81ffa5a73f1a1e4f 2024-11-28T01:42:15,529 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/2d2acedbd0c24c91a8003147104505cf is 50, key is test_row_2/B:col10/1732758134722/Put/seqid=0 2024-11-28T01:42:15,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742212_1388 (size=7415) 2024-11-28T01:42:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T01:42:15,933 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/2d2acedbd0c24c91a8003147104505cf 2024-11-28T01:42:15,939 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3dc01afed7c44d6697bb99b01ffc6cda is 50, key is test_row_2/C:col10/1732758134722/Put/seqid=0 2024-11-28T01:42:15,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742213_1389 (size=7415) 2024-11-28T01:42:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T01:42:16,343 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=628 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3dc01afed7c44d6697bb99b01ffc6cda 2024-11-28T01:42:16,347 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/A/6c4318cd42804ded81ffa5a73f1a1e4f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/6c4318cd42804ded81ffa5a73f1a1e4f 2024-11-28T01:42:16,350 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/6c4318cd42804ded81ffa5a73f1a1e4f, entries=50, sequenceid=628, filesize=7.2 K 2024-11-28T01:42:16,351 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/B/2d2acedbd0c24c91a8003147104505cf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/2d2acedbd0c24c91a8003147104505cf 2024-11-28T01:42:16,353 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/2d2acedbd0c24c91a8003147104505cf, entries=50, sequenceid=628, filesize=7.2 K 2024-11-28T01:42:16,354 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/.tmp/C/3dc01afed7c44d6697bb99b01ffc6cda as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3dc01afed7c44d6697bb99b01ffc6cda 2024-11-28T01:42:16,357 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3dc01afed7c44d6697bb99b01ffc6cda, entries=50, sequenceid=628, filesize=7.2 K 2024-11-28T01:42:16,357 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 30f1052f6aa70fa647eb86fc190d7c15 in 1241ms, sequenceid=628, compaction requested=false 2024-11-28T01:42:16,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/68d2a947000c4af8af29572053b0ad2d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/767d637713d34ab889d58c26503dc23b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/4bca7abc511e47e1bfead0c2cff73a49, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0aa472e7e0524a11bcc37767a1530dc0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/3dbfca7599234a669a0ee6628bb0c825, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/847df9c86f094efa8926f980acfd2990, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0cca3758f96d48418fcf52a741c1a6a3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/765f00e2542948c88943858fce07660c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a3b4ac3f6ee144f2ae77c61660a358fa, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/84d6f1bd8b2c4e199775e5740c2d2aa7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1cf96189796f4bfe9bfb3e2de8ebb591, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/86fcda6a773847b3a01094e8d7c9efc3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a2ab7bfe5db74f3398fa7aa9fc116ec0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1bc3eca21a004184bd1f8215f7220114, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/67fea92c8c7c4753a3ca457765218266, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e4e354a5860f49a58667a056247f4c15, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7f4d446686a443dd96454ddb11f00571, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/03b3e84e9d9a4ab2a616be4394a86b2b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e991f00010b04117bd5fc5ab4311af59, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e04c92bac6ed42c2a7d3e940b7940d63, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f1ad0d8246b84823aabb4d03adfcd1ea, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f6fcff0cd3814f2fb80bc7f5fd102f9c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9e228f09403844589b75549dfbf79117, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/53f7d6c516e84f8ebee3d615369f3b1b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/371e252a939549dfa59b196462c22985, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/5e5d7be8b9634885907ae4f0cd7f92e0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a887c3ea54c34b4b95d537bdcdcc43dd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/789ccf78649d41a5b117ed23b7a17d6e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/2af743b2194c4586a1d1408a6f57be96, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1b6c381d5a25492f86408979d9be05b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/08017665ba484dcea75394c34a13c2a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e72c59c7eef64e4d8826de28521f6d3f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7897fdffb57b42739e991105ba5cc4f7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9f677725e59f42eaa9fe4373f08bd894, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/95dbbfffce95451a9354cfe27577adf4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1fa4a5e633ae4f35bbae270624fece2f] to archive 2024-11-28T01:42:16,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:16,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/68d2a947000c4af8af29572053b0ad2d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/68d2a947000c4af8af29572053b0ad2d 2024-11-28T01:42:16,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/767d637713d34ab889d58c26503dc23b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/767d637713d34ab889d58c26503dc23b 2024-11-28T01:42:16,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/4bca7abc511e47e1bfead0c2cff73a49 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/4bca7abc511e47e1bfead0c2cff73a49 2024-11-28T01:42:16,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0aa472e7e0524a11bcc37767a1530dc0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0aa472e7e0524a11bcc37767a1530dc0 2024-11-28T01:42:16,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/3dbfca7599234a669a0ee6628bb0c825 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/3dbfca7599234a669a0ee6628bb0c825 2024-11-28T01:42:16,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/847df9c86f094efa8926f980acfd2990 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/847df9c86f094efa8926f980acfd2990 2024-11-28T01:42:16,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0cca3758f96d48418fcf52a741c1a6a3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/0cca3758f96d48418fcf52a741c1a6a3 2024-11-28T01:42:16,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/765f00e2542948c88943858fce07660c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/765f00e2542948c88943858fce07660c 2024-11-28T01:42:16,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a3b4ac3f6ee144f2ae77c61660a358fa to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a3b4ac3f6ee144f2ae77c61660a358fa 2024-11-28T01:42:16,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/84d6f1bd8b2c4e199775e5740c2d2aa7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/84d6f1bd8b2c4e199775e5740c2d2aa7 2024-11-28T01:42:16,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1cf96189796f4bfe9bfb3e2de8ebb591 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1cf96189796f4bfe9bfb3e2de8ebb591 2024-11-28T01:42:16,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/86fcda6a773847b3a01094e8d7c9efc3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/86fcda6a773847b3a01094e8d7c9efc3 2024-11-28T01:42:16,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a2ab7bfe5db74f3398fa7aa9fc116ec0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a2ab7bfe5db74f3398fa7aa9fc116ec0 2024-11-28T01:42:16,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1bc3eca21a004184bd1f8215f7220114 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1bc3eca21a004184bd1f8215f7220114 2024-11-28T01:42:16,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/67fea92c8c7c4753a3ca457765218266 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/67fea92c8c7c4753a3ca457765218266 2024-11-28T01:42:16,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e4e354a5860f49a58667a056247f4c15 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e4e354a5860f49a58667a056247f4c15 2024-11-28T01:42:16,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7f4d446686a443dd96454ddb11f00571 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7f4d446686a443dd96454ddb11f00571 2024-11-28T01:42:16,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/03b3e84e9d9a4ab2a616be4394a86b2b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/03b3e84e9d9a4ab2a616be4394a86b2b 2024-11-28T01:42:16,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e991f00010b04117bd5fc5ab4311af59 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e991f00010b04117bd5fc5ab4311af59 2024-11-28T01:42:16,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e04c92bac6ed42c2a7d3e940b7940d63 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e04c92bac6ed42c2a7d3e940b7940d63 2024-11-28T01:42:16,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f1ad0d8246b84823aabb4d03adfcd1ea to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f1ad0d8246b84823aabb4d03adfcd1ea 2024-11-28T01:42:16,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f6fcff0cd3814f2fb80bc7f5fd102f9c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/f6fcff0cd3814f2fb80bc7f5fd102f9c 2024-11-28T01:42:16,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9e228f09403844589b75549dfbf79117 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9e228f09403844589b75549dfbf79117 2024-11-28T01:42:16,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/53f7d6c516e84f8ebee3d615369f3b1b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/53f7d6c516e84f8ebee3d615369f3b1b 2024-11-28T01:42:16,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/371e252a939549dfa59b196462c22985 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/371e252a939549dfa59b196462c22985 2024-11-28T01:42:16,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/5e5d7be8b9634885907ae4f0cd7f92e0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/5e5d7be8b9634885907ae4f0cd7f92e0 2024-11-28T01:42:16,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a887c3ea54c34b4b95d537bdcdcc43dd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/a887c3ea54c34b4b95d537bdcdcc43dd 2024-11-28T01:42:16,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/789ccf78649d41a5b117ed23b7a17d6e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/789ccf78649d41a5b117ed23b7a17d6e 2024-11-28T01:42:16,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/2af743b2194c4586a1d1408a6f57be96 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/2af743b2194c4586a1d1408a6f57be96 2024-11-28T01:42:16,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1b6c381d5a25492f86408979d9be05b1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1b6c381d5a25492f86408979d9be05b1 2024-11-28T01:42:16,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/08017665ba484dcea75394c34a13c2a1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/08017665ba484dcea75394c34a13c2a1 2024-11-28T01:42:16,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e72c59c7eef64e4d8826de28521f6d3f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/e72c59c7eef64e4d8826de28521f6d3f 2024-11-28T01:42:16,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7897fdffb57b42739e991105ba5cc4f7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/7897fdffb57b42739e991105ba5cc4f7 2024-11-28T01:42:16,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9f677725e59f42eaa9fe4373f08bd894 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/9f677725e59f42eaa9fe4373f08bd894 2024-11-28T01:42:16,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/95dbbfffce95451a9354cfe27577adf4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/95dbbfffce95451a9354cfe27577adf4 2024-11-28T01:42:16,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1fa4a5e633ae4f35bbae270624fece2f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/1fa4a5e633ae4f35bbae270624fece2f 2024-11-28T01:42:16,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a626eb22cd544407ac58945de58175eb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/f38649add60b403d98e1887e7a1c5979, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e726beb3488f4d7480cc4c5e94b6af5c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/b67802d02ccc4452b22c33b51627741e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5fd626e7b29a444e8c1c4d880deb4a42, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9ed8a07116e740e78ec9ff53b90ff639, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c4341560124f415195dfa3e1dacaf93c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fc8180963ad347e7b1618d2d9ea40f57, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c16c83b9f3e1442caede9a235b294f8d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/4cd0ec8feb82455ab4839fc32639807d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/44773caf793c4a9581176764d4e9dd16, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0116bf4a20d84c9b9fb4e60e58632487, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/830e39eea8794f96b5ed6b6849555b3f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/934b801c92454108a852ba1da43d1a91, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/347106f8e62d4cd2a8fff3bf8796a9e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/bbe71b40b9c8490d8ebfca4d2903a799, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9a90103e8330478a9f5118a5e0aebff3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1024caff7ccf46bea5bce7bcc885360a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a771652818e54a1c99698d008224a50d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/90752d7d31974d7f86262c42bf55de39, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5faaed00f39c4622aa9bb613fc04acbf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9fcf8b4e69b447569d5190beef6378c4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e0c43f6bab214d3ca7aad29b36acae82, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36ee3532cc6940ef97505dba48370eca, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36dcaa6733ce405fa45feda86eb0c2c4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/071fee6de9aa4fa49ca002f455933666, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/973956b38c1f45448444a0db38b9cd40, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/eae3da68cf4d45d3916669b4a542ea2f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/040723eb4b3448be86f8e02fc0551353, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fb054997e1144b668c104fb41de0dbc2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9e14e92d91544f4cb42042d29b0f13cb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1cb85e79a6c849129ddfe9ab700a77d2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/792a325edc96410dacfc244c35712080, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0c540b569d7a4fc6b543998982eb08ac, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e2935e414a28448cae79c03df981d242] to archive 2024-11-28T01:42:16,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:16,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a626eb22cd544407ac58945de58175eb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a626eb22cd544407ac58945de58175eb 2024-11-28T01:42:16,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/f38649add60b403d98e1887e7a1c5979 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/f38649add60b403d98e1887e7a1c5979 2024-11-28T01:42:16,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e726beb3488f4d7480cc4c5e94b6af5c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e726beb3488f4d7480cc4c5e94b6af5c 2024-11-28T01:42:16,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/b67802d02ccc4452b22c33b51627741e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/b67802d02ccc4452b22c33b51627741e 2024-11-28T01:42:16,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5fd626e7b29a444e8c1c4d880deb4a42 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5fd626e7b29a444e8c1c4d880deb4a42 2024-11-28T01:42:16,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9ed8a07116e740e78ec9ff53b90ff639 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9ed8a07116e740e78ec9ff53b90ff639 2024-11-28T01:42:16,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c4341560124f415195dfa3e1dacaf93c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c4341560124f415195dfa3e1dacaf93c 2024-11-28T01:42:16,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fc8180963ad347e7b1618d2d9ea40f57 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fc8180963ad347e7b1618d2d9ea40f57 2024-11-28T01:42:16,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c16c83b9f3e1442caede9a235b294f8d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/c16c83b9f3e1442caede9a235b294f8d 2024-11-28T01:42:16,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/4cd0ec8feb82455ab4839fc32639807d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/4cd0ec8feb82455ab4839fc32639807d 2024-11-28T01:42:16,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/44773caf793c4a9581176764d4e9dd16 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/44773caf793c4a9581176764d4e9dd16 2024-11-28T01:42:16,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0116bf4a20d84c9b9fb4e60e58632487 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0116bf4a20d84c9b9fb4e60e58632487 2024-11-28T01:42:16,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/830e39eea8794f96b5ed6b6849555b3f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/830e39eea8794f96b5ed6b6849555b3f 2024-11-28T01:42:16,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/934b801c92454108a852ba1da43d1a91 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/934b801c92454108a852ba1da43d1a91 2024-11-28T01:42:16,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/347106f8e62d4cd2a8fff3bf8796a9e9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/347106f8e62d4cd2a8fff3bf8796a9e9 2024-11-28T01:42:16,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/bbe71b40b9c8490d8ebfca4d2903a799 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/bbe71b40b9c8490d8ebfca4d2903a799 2024-11-28T01:42:16,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9a90103e8330478a9f5118a5e0aebff3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9a90103e8330478a9f5118a5e0aebff3 2024-11-28T01:42:16,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1024caff7ccf46bea5bce7bcc885360a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1024caff7ccf46bea5bce7bcc885360a 2024-11-28T01:42:16,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a771652818e54a1c99698d008224a50d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/a771652818e54a1c99698d008224a50d 2024-11-28T01:42:16,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/90752d7d31974d7f86262c42bf55de39 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/90752d7d31974d7f86262c42bf55de39 2024-11-28T01:42:16,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5faaed00f39c4622aa9bb613fc04acbf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5faaed00f39c4622aa9bb613fc04acbf 2024-11-28T01:42:16,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9fcf8b4e69b447569d5190beef6378c4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9fcf8b4e69b447569d5190beef6378c4 2024-11-28T01:42:16,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e0c43f6bab214d3ca7aad29b36acae82 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e0c43f6bab214d3ca7aad29b36acae82 2024-11-28T01:42:16,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36ee3532cc6940ef97505dba48370eca to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36ee3532cc6940ef97505dba48370eca 2024-11-28T01:42:16,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36dcaa6733ce405fa45feda86eb0c2c4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/36dcaa6733ce405fa45feda86eb0c2c4 2024-11-28T01:42:16,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/071fee6de9aa4fa49ca002f455933666 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/071fee6de9aa4fa49ca002f455933666 2024-11-28T01:42:16,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/973956b38c1f45448444a0db38b9cd40 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/973956b38c1f45448444a0db38b9cd40 2024-11-28T01:42:16,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/eae3da68cf4d45d3916669b4a542ea2f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/eae3da68cf4d45d3916669b4a542ea2f 2024-11-28T01:42:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/040723eb4b3448be86f8e02fc0551353 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/040723eb4b3448be86f8e02fc0551353 2024-11-28T01:42:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/5a4bb2b5f00943e3bcbab4d2e2cca6d4 2024-11-28T01:42:16,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fb054997e1144b668c104fb41de0dbc2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/fb054997e1144b668c104fb41de0dbc2 2024-11-28T01:42:16,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9e14e92d91544f4cb42042d29b0f13cb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/9e14e92d91544f4cb42042d29b0f13cb 2024-11-28T01:42:16,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1cb85e79a6c849129ddfe9ab700a77d2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/1cb85e79a6c849129ddfe9ab700a77d2 2024-11-28T01:42:16,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/792a325edc96410dacfc244c35712080 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/792a325edc96410dacfc244c35712080 2024-11-28T01:42:16,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0c540b569d7a4fc6b543998982eb08ac to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0c540b569d7a4fc6b543998982eb08ac 2024-11-28T01:42:16,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e2935e414a28448cae79c03df981d242 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/e2935e414a28448cae79c03df981d242 2024-11-28T01:42:16,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/aafe5007b93449a5b40246f81b6b9b47, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/046e9618c20647fda832cf348d1534e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d1fa83079ec8454db908998f501e874c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/1d82ecea76074b86ac4f03444815aeba, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/89ae2df4137744c38c7f16c495d9a07d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ba026c0620aa4560974a38a64057c85d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/7d79461041f74edc864468da7bc6f676, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/924a80a903394732810b094e32c8b92b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c5768e4c11024f85bd548ca6301fa1c5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/24afa6247c114c82a284fcb412cffa63, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c63a08d13224c7fb8d2e96ff897c5af, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/0bcda7da91cb4764a0ee71ff84311ef4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/190cea53ba174682ab8ab83dfb7e8f15, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/cb1971b1647b44ce8af320f2bbf87a39, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9912b9c5992b41498c3ff3c56e1d0a91, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/07166c8a13f741e8b9d0483e3020c5fb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c28132a4f9e3437fbf504846abce43a1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/b6cca1209b0948e0bfa196185a1b9501, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/11400d0c42fb47c4b1f57fff1d627eff, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/836ddcbf43384bd1b4b9f7974182d877, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/800dfe508df64d5c933a97a75a9b307b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/50d718fde54a4a31aa271996f648869b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3fcd190c1ea74ca18e2ea3aee07db538, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/42b53d9c7cd145b8b79cd27226dccd13, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9be84ee0a09947618ac241339f91b159, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/16ad831cf09449f0928b5155635961ff, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/2939923b3c074689adb46f654fba66b4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ed8559b90afa4acd9366160bc8b0ed39, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/32dd22c1a1574ff98c61d388059a6054, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d72908d6f4ed45c6a23718c1864110c3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/e8a84f185f0c4189a7a167bd7b047d48, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c10994d9cce74f22b90b182c2f805861, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/f433507d972f4e86aa86561c1810e042, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c12ba54dcbb4ac3a03799e84fc6ab3c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/255f772c22134630986771f1fd0a7754] to archive 2024-11-28T01:42:16,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/aafe5007b93449a5b40246f81b6b9b47 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/aafe5007b93449a5b40246f81b6b9b47 2024-11-28T01:42:16,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/046e9618c20647fda832cf348d1534e4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/046e9618c20647fda832cf348d1534e4 2024-11-28T01:42:16,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d1fa83079ec8454db908998f501e874c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d1fa83079ec8454db908998f501e874c 2024-11-28T01:42:16,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/1d82ecea76074b86ac4f03444815aeba to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/1d82ecea76074b86ac4f03444815aeba 2024-11-28T01:42:16,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/89ae2df4137744c38c7f16c495d9a07d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/89ae2df4137744c38c7f16c495d9a07d 2024-11-28T01:42:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ba026c0620aa4560974a38a64057c85d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ba026c0620aa4560974a38a64057c85d 2024-11-28T01:42:16,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/7d79461041f74edc864468da7bc6f676 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/7d79461041f74edc864468da7bc6f676 2024-11-28T01:42:16,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/924a80a903394732810b094e32c8b92b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/924a80a903394732810b094e32c8b92b 2024-11-28T01:42:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c5768e4c11024f85bd548ca6301fa1c5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c5768e4c11024f85bd548ca6301fa1c5 2024-11-28T01:42:16,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/24afa6247c114c82a284fcb412cffa63 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/24afa6247c114c82a284fcb412cffa63 2024-11-28T01:42:16,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c63a08d13224c7fb8d2e96ff897c5af to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c63a08d13224c7fb8d2e96ff897c5af 2024-11-28T01:42:16,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/0bcda7da91cb4764a0ee71ff84311ef4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/0bcda7da91cb4764a0ee71ff84311ef4 2024-11-28T01:42:16,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/190cea53ba174682ab8ab83dfb7e8f15 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/190cea53ba174682ab8ab83dfb7e8f15 2024-11-28T01:42:16,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/cb1971b1647b44ce8af320f2bbf87a39 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/cb1971b1647b44ce8af320f2bbf87a39 2024-11-28T01:42:16,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9912b9c5992b41498c3ff3c56e1d0a91 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9912b9c5992b41498c3ff3c56e1d0a91 2024-11-28T01:42:16,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/07166c8a13f741e8b9d0483e3020c5fb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/07166c8a13f741e8b9d0483e3020c5fb 2024-11-28T01:42:16,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c28132a4f9e3437fbf504846abce43a1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c28132a4f9e3437fbf504846abce43a1 2024-11-28T01:42:16,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/b6cca1209b0948e0bfa196185a1b9501 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/b6cca1209b0948e0bfa196185a1b9501 2024-11-28T01:42:16,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/11400d0c42fb47c4b1f57fff1d627eff to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/11400d0c42fb47c4b1f57fff1d627eff 2024-11-28T01:42:16,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/836ddcbf43384bd1b4b9f7974182d877 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/836ddcbf43384bd1b4b9f7974182d877 2024-11-28T01:42:16,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/800dfe508df64d5c933a97a75a9b307b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/800dfe508df64d5c933a97a75a9b307b 2024-11-28T01:42:16,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/50d718fde54a4a31aa271996f648869b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/50d718fde54a4a31aa271996f648869b 2024-11-28T01:42:16,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3fcd190c1ea74ca18e2ea3aee07db538 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3fcd190c1ea74ca18e2ea3aee07db538 2024-11-28T01:42:16,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/42b53d9c7cd145b8b79cd27226dccd13 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/42b53d9c7cd145b8b79cd27226dccd13 2024-11-28T01:42:16,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9be84ee0a09947618ac241339f91b159 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/9be84ee0a09947618ac241339f91b159 2024-11-28T01:42:16,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/16ad831cf09449f0928b5155635961ff to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/16ad831cf09449f0928b5155635961ff 2024-11-28T01:42:16,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/2939923b3c074689adb46f654fba66b4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/2939923b3c074689adb46f654fba66b4 2024-11-28T01:42:16,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ed8559b90afa4acd9366160bc8b0ed39 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/ed8559b90afa4acd9366160bc8b0ed39 2024-11-28T01:42:16,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/32dd22c1a1574ff98c61d388059a6054 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/32dd22c1a1574ff98c61d388059a6054 2024-11-28T01:42:16,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d72908d6f4ed45c6a23718c1864110c3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/d72908d6f4ed45c6a23718c1864110c3 2024-11-28T01:42:16,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/e8a84f185f0c4189a7a167bd7b047d48 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/e8a84f185f0c4189a7a167bd7b047d48 2024-11-28T01:42:16,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c10994d9cce74f22b90b182c2f805861 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/c10994d9cce74f22b90b182c2f805861 2024-11-28T01:42:16,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/f433507d972f4e86aa86561c1810e042 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/f433507d972f4e86aa86561c1810e042 2024-11-28T01:42:16,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c12ba54dcbb4ac3a03799e84fc6ab3c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/4c12ba54dcbb4ac3a03799e84fc6ab3c 2024-11-28T01:42:16,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/255f772c22134630986771f1fd0a7754 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/255f772c22134630986771f1fd0a7754 2024-11-28T01:42:16,468 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/recovered.edits/631.seqid, newMaxSeqId=631, maxSeqId=1 2024-11-28T01:42:16,468 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. 2024-11-28T01:42:16,468 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 30f1052f6aa70fa647eb86fc190d7c15: Waiting for close lock at 1732758135116Running coprocessor pre-close hooks at 1732758135116Disabling compacts and flushes for region at 1732758135116Disabling writes for close at 1732758135116Obtaining lock to block concurrent updates at 1732758135116Preparing flush snapshotting stores in 30f1052f6aa70fa647eb86fc190d7c15 at 1732758135116Finished memstore snapshotting TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732758135116Flushing stores of TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15. at 1732758135117 (+1 ms)Flushing 30f1052f6aa70fa647eb86fc190d7c15/A: creating writer at 1732758135117Flushing 30f1052f6aa70fa647eb86fc190d7c15/A: appending metadata at 1732758135119 (+2 ms)Flushing 30f1052f6aa70fa647eb86fc190d7c15/A: closing flushed file at 1732758135119Flushing 30f1052f6aa70fa647eb86fc190d7c15/B: creating writer at 1732758135527 (+408 ms)Flushing 30f1052f6aa70fa647eb86fc190d7c15/B: appending metadata at 1732758135529 (+2 ms)Flushing 30f1052f6aa70fa647eb86fc190d7c15/B: closing flushed file at 1732758135529Flushing 30f1052f6aa70fa647eb86fc190d7c15/C: creating writer at 1732758135937 (+408 ms)Flushing 30f1052f6aa70fa647eb86fc190d7c15/C: appending metadata at 1732758135939 (+2 ms)Flushing 30f1052f6aa70fa647eb86fc190d7c15/C: closing flushed file at 1732758135939Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7161d111: reopening flushed file at 1732758136346 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@618c8d62: reopening flushed file at 1732758136350 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@184e7a34: reopening flushed file at 1732758136354 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 30f1052f6aa70fa647eb86fc190d7c15 in 1241ms, sequenceid=628, compaction requested=false at 1732758136358 (+4 ms)Writing region close event to WAL at 1732758136465 (+107 ms)Running coprocessor post-close hooks at 1732758136468 (+3 ms)Closed at 1732758136468 2024-11-28T01:42:16,471 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:16,471 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=30f1052f6aa70fa647eb86fc190d7c15, regionState=CLOSED 2024-11-28T01:42:16,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:42:16,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-28T01:42:16,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 30f1052f6aa70fa647eb86fc190d7c15, server=9df3f25f5867,42371,1732758050017 in 1.5110 sec 2024-11-28T01:42:16,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-28T01:42:16,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30f1052f6aa70fa647eb86fc190d7c15, UNASSIGN in 1.5150 sec 2024-11-28T01:42:16,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-28T01:42:16,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5190 sec 2024-11-28T01:42:16,481 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758136481"}]},"ts":"1732758136481"} 2024-11-28T01:42:16,482 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T01:42:16,482 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T01:42:16,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5290 sec 2024-11-28T01:42:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T01:42:17,103 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:17,104 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T01:42:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:17,105 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T01:42:17,106 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:17,109 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:17,110 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/recovered.edits] 2024-11-28T01:42:17,112 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/6c4318cd42804ded81ffa5a73f1a1e4f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/6c4318cd42804ded81ffa5a73f1a1e4f 2024-11-28T01:42:17,113 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/daca8ed12491411ab2ba3bde37f70f20 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/A/daca8ed12491411ab2ba3bde37f70f20 2024-11-28T01:42:17,115 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0042d7eb11d342cf879b8dbe99693648 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/0042d7eb11d342cf879b8dbe99693648 2024-11-28T01:42:17,116 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/2d2acedbd0c24c91a8003147104505cf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/B/2d2acedbd0c24c91a8003147104505cf 2024-11-28T01:42:17,117 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3a2ec7c5ecb74c8cacbf3a8ba11e7dfb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3a2ec7c5ecb74c8cacbf3a8ba11e7dfb 2024-11-28T01:42:17,118 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3dc01afed7c44d6697bb99b01ffc6cda to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/C/3dc01afed7c44d6697bb99b01ffc6cda 2024-11-28T01:42:17,121 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/recovered.edits/631.seqid to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15/recovered.edits/631.seqid 2024-11-28T01:42:17,122 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/30f1052f6aa70fa647eb86fc190d7c15 2024-11-28T01:42:17,124 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T01:42:17,126 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:17,128 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T01:42:17,130 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T01:42:17,131 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:17,131 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T01:42:17,131 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732758137131"}]},"ts":"9223372036854775807"} 2024-11-28T01:42:17,133 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T01:42:17,133 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 30f1052f6aa70fa647eb86fc190d7c15, NAME => 'TestAcidGuarantees,,1732758110734.30f1052f6aa70fa647eb86fc190d7c15.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T01:42:17,133 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T01:42:17,133 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732758137133"}]},"ts":"9223372036854775807"} 2024-11-28T01:42:17,135 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T01:42:17,135 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:17,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-28T01:42:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T01:42:17,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T01:42:17,213 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:17,223 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_752253615_22 at /127.0.0.1:48020 [Waiting for operation #308] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1056209077_22 at /127.0.0.1:42372 [Waiting for operation #398] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=641 (was 571) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7216 (was 7501) 2024-11-28T01:42:17,232 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=641, ProcessCount=11, AvailableMemoryMB=7216 2024-11-28T01:42:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:42:17,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:42:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:17,236 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:42:17,236 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:17,236 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-28T01:42:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T01:42:17,237 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:42:17,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742214_1390 (size=1061) 2024-11-28T01:42:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T01:42:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T01:42:17,643 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:42:17,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742215_1391 (size=53) 2024-11-28T01:42:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T01:42:18,019 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T01:42:18,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:42:18,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9420be97650ba7f85aad07a9b4f2cb6f, disabling compactions & flushes 2024-11-28T01:42:18,049 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. after waiting 0 ms 2024-11-28T01:42:18,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,049 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,049 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9420be97650ba7f85aad07a9b4f2cb6f: Waiting for close lock at 1732758138049Disabling compacts and flushes for region at 1732758138049Disabling writes for close at 1732758138049Writing region close event to WAL at 1732758138049Closed at 1732758138049 2024-11-28T01:42:18,050 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:42:18,050 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732758138050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758138050"}]},"ts":"1732758138050"} 2024-11-28T01:42:18,051 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:42:18,052 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:42:18,052 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758138052"}]},"ts":"1732758138052"} 2024-11-28T01:42:18,053 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T01:42:18,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, ASSIGN}] 2024-11-28T01:42:18,055 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, ASSIGN 2024-11-28T01:42:18,055 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, ASSIGN; state=OFFLINE, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=false 2024-11-28T01:42:18,206 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:18,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, ASSIGN because future has completed 2024-11-28T01:42:18,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:42:18,362 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,362 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:42:18,362 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,362 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:42:18,362 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,362 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,363 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,365 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:18,365 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9420be97650ba7f85aad07a9b4f2cb6f columnFamilyName A 2024-11-28T01:42:18,365 DEBUG [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:18,365 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(327): Store=9420be97650ba7f85aad07a9b4f2cb6f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:18,365 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,366 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:18,366 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9420be97650ba7f85aad07a9b4f2cb6f columnFamilyName B 2024-11-28T01:42:18,366 DEBUG [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:18,366 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(327): Store=9420be97650ba7f85aad07a9b4f2cb6f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:18,367 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,367 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:18,367 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9420be97650ba7f85aad07a9b4f2cb6f columnFamilyName C 2024-11-28T01:42:18,367 DEBUG [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:18,368 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(327): Store=9420be97650ba7f85aad07a9b4f2cb6f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:18,368 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,368 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,369 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,369 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,370 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,370 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,370 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:42:18,371 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T01:42:18,372 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:42:18,373 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 9420be97650ba7f85aad07a9b4f2cb6f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64936698, jitterRate=-0.03236779570579529}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:42:18,373 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:18,374 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 9420be97650ba7f85aad07a9b4f2cb6f: Running coprocessor pre-open hook at 1732758138362Writing region info on filesystem at 1732758138362Initializing all the Stores at 1732758138363 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758138363Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758138363Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758138363Cleaning up temporary data from old regions at 1732758138370 (+7 ms)Running coprocessor post-open hooks at 1732758138373 (+3 ms)Region opened successfully at 1732758138373 2024-11-28T01:42:18,374 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., pid=91, masterSystemTime=1732758138359 2024-11-28T01:42:18,376 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,376 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:18,376 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:18,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:42:18,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-28T01:42:18,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 in 170 msec 2024-11-28T01:42:18,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-28T01:42:18,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, ASSIGN in 326 msec 2024-11-28T01:42:18,382 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:42:18,382 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758138382"}]},"ts":"1732758138382"} 2024-11-28T01:42:18,383 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T01:42:18,384 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:42:18,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-28T01:42:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-28T01:42:19,393 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:19,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed6dd65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:19,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:19,395 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:19,396 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31543fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:19,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:19,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:19,398 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a5bc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:19,399 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:19,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:19,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:42:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,45903,1732758049085 2024-11-28T01:42:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4bcadf8 2024-11-28T01:42:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:42:19,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:42:19,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:42:19,406 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:42:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:19,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742216_1392 (size=1097) 2024-11-28T01:42:19,819 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T01:42:19,819 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T01:42:19,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:42:19,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, REOPEN/MOVE}] 2024-11-28T01:42:19,825 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, REOPEN/MOVE 2024-11-28T01:42:19,825 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:19,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, REOPEN/MOVE because future has completed 2024-11-28T01:42:19,827 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:42:19,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:42:19,979 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:19,979 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:42:19,979 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 9420be97650ba7f85aad07a9b4f2cb6f, disabling compactions & flushes 2024-11-28T01:42:19,979 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:19,979 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:19,979 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. after waiting 0 ms 2024-11-28T01:42:19,979 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:19,983 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T01:42:19,983 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:19,983 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 9420be97650ba7f85aad07a9b4f2cb6f: Waiting for close lock at 1732758139979Running coprocessor pre-close hooks at 1732758139979Disabling compacts and flushes for region at 1732758139979Disabling writes for close at 1732758139979Writing region close event to WAL at 1732758139980 (+1 ms)Running coprocessor post-close hooks at 1732758139983 (+3 ms)Closed at 1732758139983 2024-11-28T01:42:19,983 WARN [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 9420be97650ba7f85aad07a9b4f2cb6f to self. 2024-11-28T01:42:19,985 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:19,985 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=CLOSED 2024-11-28T01:42:19,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:42:19,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-28T01:42:19,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 in 161 msec 2024-11-28T01:42:19,990 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, REOPEN/MOVE; state=CLOSED, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=true 2024-11-28T01:42:20,141 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, REOPEN/MOVE because future has completed 2024-11-28T01:42:20,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:42:20,297 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,297 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:42:20,297 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,297 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:42:20,297 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,297 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,299 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,299 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:20,300 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9420be97650ba7f85aad07a9b4f2cb6f columnFamilyName A 2024-11-28T01:42:20,301 DEBUG [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:20,301 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(327): Store=9420be97650ba7f85aad07a9b4f2cb6f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:20,301 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,302 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:20,302 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9420be97650ba7f85aad07a9b4f2cb6f columnFamilyName B 2024-11-28T01:42:20,302 DEBUG [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:20,302 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(327): Store=9420be97650ba7f85aad07a9b4f2cb6f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:20,302 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,303 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:20,303 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9420be97650ba7f85aad07a9b4f2cb6f columnFamilyName C 2024-11-28T01:42:20,303 DEBUG [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:20,303 INFO [StoreOpener-9420be97650ba7f85aad07a9b4f2cb6f-1 {}] regionserver.HStore(327): Store=9420be97650ba7f85aad07a9b4f2cb6f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:20,303 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,303 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,304 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,305 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,306 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,306 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,306 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:42:20,307 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,307 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 9420be97650ba7f85aad07a9b4f2cb6f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67349724, jitterRate=0.0035890936851501465}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:42:20,308 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,308 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 9420be97650ba7f85aad07a9b4f2cb6f: Running coprocessor pre-open hook at 1732758140298Writing region info on filesystem at 1732758140298Initializing all the Stores at 1732758140298Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758140298Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758140298Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758140298Cleaning up temporary data from old regions at 1732758140306 (+8 ms)Running coprocessor post-open hooks at 1732758140308 (+2 ms)Region opened successfully at 1732758140308 2024-11-28T01:42:20,309 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., pid=96, masterSystemTime=1732758140294 2024-11-28T01:42:20,311 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,311 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=OPEN, openSeqNum=5, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:42:20,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-28T01:42:20,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 in 171 msec 2024-11-28T01:42:20,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-28T01:42:20,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, REOPEN/MOVE in 491 msec 2024-11-28T01:42:20,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-28T01:42:20,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 495 msec 2024-11-28T01:42:20,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-11-28T01:42:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T01:42:20,321 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:20,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,323 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d22cff0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc2c340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,327 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae54fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208a40a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,330 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7076ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,331 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e66061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,333 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c57ac89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164a44c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,336 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3b0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,337 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@858c873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,339 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652c8a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,340 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e61531e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,342 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291cd5b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,343 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5635268e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,345 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416f8dbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,346 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f576c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,348 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4067e73d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,349 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531724d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:20,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:20,351 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d32cf09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:20,352 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40499e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:20,353 DEBUG [Thread-1768 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,353 DEBUG [Thread-1753 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,353 DEBUG [Thread-1766 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,356 DEBUG [Thread-1772 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,357 DEBUG [Thread-1755 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,359 DEBUG [Thread-1770 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,360 DEBUG [Thread-1761 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,362 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:20,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-28T01:42:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T01:42:20,368 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:20,371 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:20,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:20,372 DEBUG [Thread-1757 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,373 DEBUG [Thread-1757 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,373 DEBUG [Thread-1757 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,374 DEBUG [Thread-1764 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,374 DEBUG [Thread-1759 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,376 DEBUG [Thread-1759 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,383 DEBUG [Thread-1764 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:20,383 DEBUG [Thread-1764 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T01:42:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:42:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:20,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f5cb5e8f0f144cdbeab7ca9be242bb5_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758140371/Put/seqid=0 2024-11-28T01:42:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758200434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758200434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758200434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742217_1393 (size=17034) 2024-11-28T01:42:20,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758200441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758200442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T01:42:20,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T01:42:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758200552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758200553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758200553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758200562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758200562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T01:42:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T01:42:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758200772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758200772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758200772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758200783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758200783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:20,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T01:42:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,853 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:20,857 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f5cb5e8f0f144cdbeab7ca9be242bb5_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f5cb5e8f0f144cdbeab7ca9be242bb5_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:20,858 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/05b3177f60f84ee4ab65aabd564c8135, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:20,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/05b3177f60f84ee4ab65aabd564c8135 is 175, key is test_row_0/A:col10/1732758140371/Put/seqid=0 2024-11-28T01:42:20,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742218_1394 (size=48139) 2024-11-28T01:42:20,871 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/05b3177f60f84ee4ab65aabd564c8135 2024-11-28T01:42:20,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/ca2c2da397b740b1bd73570ce483b527 is 50, key is test_row_0/B:col10/1732758140371/Put/seqid=0 2024-11-28T01:42:20,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742219_1395 (size=12001) 2024-11-28T01:42:20,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T01:42:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:20,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T01:42:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758201083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758201083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758201092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758201102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758201103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T01:42:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:21,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:21,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:21,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T01:42:21,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:21,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:21,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:21,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:21,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:21,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/ca2c2da397b740b1bd73570ce483b527 2024-11-28T01:42:21,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/94d3c483be764e36b2e781c6d829eb93 is 50, key is test_row_0/C:col10/1732758140371/Put/seqid=0 2024-11-28T01:42:21,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742220_1396 (size=12001) 2024-11-28T01:42:21,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/94d3c483be764e36b2e781c6d829eb93 2024-11-28T01:42:21,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/05b3177f60f84ee4ab65aabd564c8135 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/05b3177f60f84ee4ab65aabd564c8135 2024-11-28T01:42:21,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/05b3177f60f84ee4ab65aabd564c8135, entries=250, sequenceid=17, filesize=47.0 K 2024-11-28T01:42:21,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/ca2c2da397b740b1bd73570ce483b527 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/ca2c2da397b740b1bd73570ce483b527 2024-11-28T01:42:21,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/ca2c2da397b740b1bd73570ce483b527, entries=150, sequenceid=17, filesize=11.7 K 2024-11-28T01:42:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/94d3c483be764e36b2e781c6d829eb93 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/94d3c483be764e36b2e781c6d829eb93 2024-11-28T01:42:21,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/94d3c483be764e36b2e781c6d829eb93, entries=150, sequenceid=17, filesize=11.7 K 2024-11-28T01:42:21,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9420be97650ba7f85aad07a9b4f2cb6f in 979ms, sequenceid=17, compaction requested=false 2024-11-28T01:42:21,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:21,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-28T01:42:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:21,445 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:42:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b145d5c49fa64e84a88bd6902a505af8_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758140431/Put/seqid=0 2024-11-28T01:42:21,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742221_1397 (size=12154) 2024-11-28T01:42:21,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:21,460 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b145d5c49fa64e84a88bd6902a505af8_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b145d5c49fa64e84a88bd6902a505af8_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:21,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/78da5ee50d6e4039a670671c4c269e86, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:21,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/78da5ee50d6e4039a670671c4c269e86 is 175, key is test_row_0/A:col10/1732758140431/Put/seqid=0 2024-11-28T01:42:21,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742222_1398 (size=30955) 2024-11-28T01:42:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T01:42:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:21,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758201626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758201633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758201637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758201637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758201637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758201752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758201752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758201752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758201752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,869 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/78da5ee50d6e4039a670671c4c269e86 2024-11-28T01:42:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/aa9e2069f9cc4b36aa18f3a6774e61e6 is 50, key is test_row_0/B:col10/1732758140431/Put/seqid=0 2024-11-28T01:42:21,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742223_1399 (size=12001) 2024-11-28T01:42:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758201972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758201972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758201973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758201973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:21,994 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:42:22,281 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/aa9e2069f9cc4b36aa18f3a6774e61e6 2024-11-28T01:42:22,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/3b2b3d7bd0ab48579284f97814fa40b0 is 50, key is test_row_0/C:col10/1732758140431/Put/seqid=0 2024-11-28T01:42:22,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758202282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758202282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758202282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758202293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742224_1400 (size=12001) 2024-11-28T01:42:22,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/3b2b3d7bd0ab48579284f97814fa40b0 2024-11-28T01:42:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/78da5ee50d6e4039a670671c4c269e86 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/78da5ee50d6e4039a670671c4c269e86 2024-11-28T01:42:22,306 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/78da5ee50d6e4039a670671c4c269e86, entries=150, sequenceid=36, filesize=30.2 K 2024-11-28T01:42:22,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/aa9e2069f9cc4b36aa18f3a6774e61e6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/aa9e2069f9cc4b36aa18f3a6774e61e6 2024-11-28T01:42:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,314 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/aa9e2069f9cc4b36aa18f3a6774e61e6, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T01:42:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/3b2b3d7bd0ab48579284f97814fa40b0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3b2b3d7bd0ab48579284f97814fa40b0 2024-11-28T01:42:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,320 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3b2b3d7bd0ab48579284f97814fa40b0, entries=150, sequenceid=36, filesize=11.7 K 2024-11-28T01:42:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,321 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9420be97650ba7f85aad07a9b4f2cb6f in 877ms, sequenceid=36, compaction requested=false 2024-11-28T01:42:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-28T01:42:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-28T01:42:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-28T01:42:22,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9520 sec 2024-11-28T01:42:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.9640 sec 2024-11-28T01:42:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-28T01:42:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,513 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,514 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-28T01:42:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T01:42:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T01:42:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-28T01:42:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,672 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:42:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128728248d0aa7f4179aea0a694efd3267c_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758141634/Put/seqid=0 2024-11-28T01:42:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742225_1401 (size=9714) 2024-11-28T01:42:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:22,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758202816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758202820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758202821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T01:42:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758202827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758202829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758202933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758202933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758202942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758202942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758202943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T01:42:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:23,150 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128728248d0aa7f4179aea0a694efd3267c_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128728248d0aa7f4179aea0a694efd3267c_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1c098bcbf1e146c1983b4355931fafe5, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:23,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1c098bcbf1e146c1983b4355931fafe5 is 175, key is test_row_0/A:col10/1732758141634/Put/seqid=0 2024-11-28T01:42:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758203153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742226_1402 (size=22361) 2024-11-28T01:42:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758203153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758203153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758203153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758203153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758203472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758203472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758203473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758203473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758203473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:23,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:23,558 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1c098bcbf1e146c1983b4355931fafe5 2024-11-28T01:42:23,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/c274595760ac44eeb28d1b1bbd714960 is 50, key is test_row_0/B:col10/1732758141634/Put/seqid=0 2024-11-28T01:42:23,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742227_1403 (size=9657) 2024-11-28T01:42:23,595 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/c274595760ac44eeb28d1b1bbd714960 2024-11-28T01:42:23,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/dc90df4c06f642fabc395ad3043d13ca is 50, key is test_row_0/C:col10/1732758141634/Put/seqid=0 2024-11-28T01:42:23,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742228_1404 (size=9657) 2024-11-28T01:42:23,624 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/dc90df4c06f642fabc395ad3043d13ca 2024-11-28T01:42:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1c098bcbf1e146c1983b4355931fafe5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1c098bcbf1e146c1983b4355931fafe5 2024-11-28T01:42:23,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1c098bcbf1e146c1983b4355931fafe5, entries=100, sequenceid=49, filesize=21.8 K 2024-11-28T01:42:23,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/c274595760ac44eeb28d1b1bbd714960 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c274595760ac44eeb28d1b1bbd714960 2024-11-28T01:42:23,646 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c274595760ac44eeb28d1b1bbd714960, entries=100, sequenceid=49, filesize=9.4 K 2024-11-28T01:42:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/dc90df4c06f642fabc395ad3043d13ca as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dc90df4c06f642fabc395ad3043d13ca 2024-11-28T01:42:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T01:42:23,661 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dc90df4c06f642fabc395ad3043d13ca, entries=100, sequenceid=49, filesize=9.4 K 2024-11-28T01:42:23,662 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9420be97650ba7f85aad07a9b4f2cb6f in 990ms, sequenceid=49, compaction requested=true 2024-11-28T01:42:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:23,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-28T01:42:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-28T01:42:23,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-28T01:42:23,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-11-28T01:42:23,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1520 sec 2024-11-28T01:42:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:23,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:42:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:23,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:24,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281c09570b79544c9a9ba891d5b1f21498_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758142820/Put/seqid=0 2024-11-28T01:42:24,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742229_1405 (size=14594) 2024-11-28T01:42:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758204014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758204016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758204018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758204019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758204020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758204123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758204132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758204132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758204142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758204142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758204333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758204352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758204352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758204352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758204352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:24,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281c09570b79544c9a9ba891d5b1f21498_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281c09570b79544c9a9ba891d5b1f21498_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:24,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/bfded09129e74d26aecc6ec312a845e7, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:24,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/bfded09129e74d26aecc6ec312a845e7 is 175, key is test_row_0/A:col10/1732758142820/Put/seqid=0 2024-11-28T01:42:24,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742230_1406 (size=39549) 2024-11-28T01:42:24,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758204642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-28T01:42:24,663 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:24,663 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-28T01:42:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T01:42:24,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:24,667 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:24,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:24,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758204662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758204673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758204673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758204673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T01:42:24,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T01:42:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:24,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:24,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:24,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:24,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:24,829 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/bfded09129e74d26aecc6ec312a845e7 2024-11-28T01:42:24,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/def9cca553724a9594897d5a00897542 is 50, key is test_row_0/B:col10/1732758142820/Put/seqid=0 2024-11-28T01:42:24,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742231_1407 (size=12001) 2024-11-28T01:42:24,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T01:42:24,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:24,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:24,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:24,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:24,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T01:42:25,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T01:42:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:25,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758205153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:25,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758205182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:25,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758205193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:25,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758205193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:25,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758205193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:25,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/def9cca553724a9594897d5a00897542 2024-11-28T01:42:25,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/9f3e95f06f2946cab14e4d3ed24daeb8 is 50, key is test_row_0/C:col10/1732758142820/Put/seqid=0 2024-11-28T01:42:25,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742232_1408 (size=12001) 2024-11-28T01:42:25,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/9f3e95f06f2946cab14e4d3ed24daeb8 2024-11-28T01:42:25,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/bfded09129e74d26aecc6ec312a845e7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bfded09129e74d26aecc6ec312a845e7 2024-11-28T01:42:25,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bfded09129e74d26aecc6ec312a845e7, entries=200, sequenceid=69, filesize=38.6 K 2024-11-28T01:42:25,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/def9cca553724a9594897d5a00897542 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/def9cca553724a9594897d5a00897542 2024-11-28T01:42:25,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/def9cca553724a9594897d5a00897542, entries=150, sequenceid=69, filesize=11.7 K 2024-11-28T01:42:25,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/9f3e95f06f2946cab14e4d3ed24daeb8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9f3e95f06f2946cab14e4d3ed24daeb8 2024-11-28T01:42:25,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T01:42:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:25,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9f3e95f06f2946cab14e4d3ed24daeb8, entries=150, sequenceid=69, filesize=11.7 K 2024-11-28T01:42:25,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9420be97650ba7f85aad07a9b4f2cb6f in 1282ms, sequenceid=69, compaction requested=true 2024-11-28T01:42:25,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:25,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:25,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:25,282 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:25,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:25,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:25,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:25,282 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:25,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:25,283 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:25,283 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:25,283 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,283 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/05b3177f60f84ee4ab65aabd564c8135, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/78da5ee50d6e4039a670671c4c269e86, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1c098bcbf1e146c1983b4355931fafe5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bfded09129e74d26aecc6ec312a845e7] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=137.7 K 2024-11-28T01:42:25,283 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,283 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/05b3177f60f84ee4ab65aabd564c8135, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/78da5ee50d6e4039a670671c4c269e86, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1c098bcbf1e146c1983b4355931fafe5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bfded09129e74d26aecc6ec312a845e7] 2024-11-28T01:42:25,284 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:25,284 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:25,284 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,284 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/ca2c2da397b740b1bd73570ce483b527, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/aa9e2069f9cc4b36aa18f3a6774e61e6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c274595760ac44eeb28d1b1bbd714960, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/def9cca553724a9594897d5a00897542] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=44.6 K 2024-11-28T01:42:25,284 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2c2da397b740b1bd73570ce483b527, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732758140371 2024-11-28T01:42:25,284 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b3177f60f84ee4ab65aabd564c8135, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732758140371 2024-11-28T01:42:25,285 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78da5ee50d6e4039a670671c4c269e86, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732758140430 2024-11-28T01:42:25,285 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9e2069f9cc4b36aa18f3a6774e61e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732758140430 2024-11-28T01:42:25,285 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c274595760ac44eeb28d1b1bbd714960, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732758141633 2024-11-28T01:42:25,285 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c098bcbf1e146c1983b4355931fafe5, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732758141633 2024-11-28T01:42:25,285 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfded09129e74d26aecc6ec312a845e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732758142818 2024-11-28T01:42:25,285 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting def9cca553724a9594897d5a00897542, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732758142818 2024-11-28T01:42:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T01:42:25,301 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:25,303 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:25,304 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/6eb7722538594ab9a099421fed8c5235 is 50, key is test_row_0/B:col10/1732758142820/Put/seqid=0 2024-11-28T01:42:25,311 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411283b0bd781be334ce98976c061ec9ce40f_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:25,316 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411283b0bd781be334ce98976c061ec9ce40f_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:25,316 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283b0bd781be334ce98976c061ec9ce40f_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:25,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742233_1409 (size=12139) 2024-11-28T01:42:25,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742234_1410 (size=4469) 2024-11-28T01:42:25,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T01:42:25,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:42:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287e878a9c6abb4095b19f7a5c0b50fcd4_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758144015/Put/seqid=0 2024-11-28T01:42:25,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742235_1411 (size=12154) 2024-11-28T01:42:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:25,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287e878a9c6abb4095b19f7a5c0b50fcd4_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287e878a9c6abb4095b19f7a5c0b50fcd4_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:25,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d192eabe277147d8bf455458e0aa87ab, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:25,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d192eabe277147d8bf455458e0aa87ab is 175, key is test_row_0/A:col10/1732758144015/Put/seqid=0 2024-11-28T01:42:25,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742236_1412 (size=30955) 2024-11-28T01:42:25,722 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/6eb7722538594ab9a099421fed8c5235 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/6eb7722538594ab9a099421fed8c5235 2024-11-28T01:42:25,727 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into 6eb7722538594ab9a099421fed8c5235(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:25,727 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:25,727 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=12, startTime=1732758145282; duration=0sec 2024-11-28T01:42:25,727 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:25,727 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:25,727 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:25,729 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:25,729 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:25,729 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:25,729 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/94d3c483be764e36b2e781c6d829eb93, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3b2b3d7bd0ab48579284f97814fa40b0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dc90df4c06f642fabc395ad3043d13ca, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9f3e95f06f2946cab14e4d3ed24daeb8] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=44.6 K 2024-11-28T01:42:25,730 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d3c483be764e36b2e781c6d829eb93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732758140371 2024-11-28T01:42:25,731 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2b3d7bd0ab48579284f97814fa40b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732758140430 2024-11-28T01:42:25,731 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting dc90df4c06f642fabc395ad3043d13ca, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732758141633 2024-11-28T01:42:25,732 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3e95f06f2946cab14e4d3ed24daeb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732758142818 2024-11-28T01:42:25,738 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#350 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:25,739 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/c4dc6c8e3d25497f96a80635d76b9778 is 175, key is test_row_0/A:col10/1732758142820/Put/seqid=0 2024-11-28T01:42:25,742 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:25,743 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/0088d599e4834a93a40045f4c6ddcc7e is 50, key is test_row_0/C:col10/1732758142820/Put/seqid=0 2024-11-28T01:42:25,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742237_1413 (size=31093) 2024-11-28T01:42:25,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742238_1414 (size=12139) 2024-11-28T01:42:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T01:42:25,860 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d192eabe277147d8bf455458e0aa87ab 2024-11-28T01:42:25,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/74fff2bbaae14beb9bec5d9cb15242eb is 50, key is test_row_0/B:col10/1732758144015/Put/seqid=0 2024-11-28T01:42:25,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742239_1415 (size=12001) 2024-11-28T01:42:26,162 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/c4dc6c8e3d25497f96a80635d76b9778 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/c4dc6c8e3d25497f96a80635d76b9778 2024-11-28T01:42:26,167 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/0088d599e4834a93a40045f4c6ddcc7e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/0088d599e4834a93a40045f4c6ddcc7e 2024-11-28T01:42:26,170 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into c4dc6c8e3d25497f96a80635d76b9778(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:26,170 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:26,170 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=12, startTime=1732758145281; duration=0sec 2024-11-28T01:42:26,170 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:26,170 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:26,172 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into 0088d599e4834a93a40045f4c6ddcc7e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:26,172 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:26,172 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=12, startTime=1732758145282; duration=0sec 2024-11-28T01:42:26,173 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:26,173 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:26,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:26,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758206224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758206228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758146221, max timestamp=1732758146240Num uniques 859; 2024-11-28T01:42:26,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758146221, max timestamp=1732758146240Num uniques 859; 2024-11-28T01:42:26,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:42:26,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:42:26,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732758146226, max timestamp=1732758146240Num uniques 859; 2024-11-28T01:42:26,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T01:42:26,274 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/74fff2bbaae14beb9bec5d9cb15242eb 2024-11-28T01:42:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/8578c06b05734b259c0d3c313acbf2d7 is 50, key is test_row_0/C:col10/1732758144015/Put/seqid=0 2024-11-28T01:42:26,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742240_1416 (size=12001) 2024-11-28T01:42:26,287 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/8578c06b05734b259c0d3c313acbf2d7 2024-11-28T01:42:26,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d192eabe277147d8bf455458e0aa87ab as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d192eabe277147d8bf455458e0aa87ab 2024-11-28T01:42:26,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d192eabe277147d8bf455458e0aa87ab, entries=150, sequenceid=81, filesize=30.2 K 2024-11-28T01:42:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/74fff2bbaae14beb9bec5d9cb15242eb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/74fff2bbaae14beb9bec5d9cb15242eb 2024-11-28T01:42:26,304 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/74fff2bbaae14beb9bec5d9cb15242eb, entries=150, sequenceid=81, filesize=11.7 K 2024-11-28T01:42:26,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-28T01:42:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/8578c06b05734b259c0d3c313acbf2d7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/8578c06b05734b259c0d3c313acbf2d7 2024-11-28T01:42:26,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758206295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758206303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,314 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/8578c06b05734b259c0d3c313acbf2d7, entries=150, sequenceid=81, filesize=11.7 K 2024-11-28T01:42:26,316 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=122.60 KB/125545 for 9420be97650ba7f85aad07a9b4f2cb6f in 884ms, sequenceid=81, compaction requested=false 2024-11-28T01:42:26,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:26,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:26,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-28T01:42:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-28T01:42:26,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-28T01:42:26,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6500 sec 2024-11-28T01:42:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:26,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-11-28T01:42:26,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:26,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:26,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:26,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:26,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:26,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:26,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.6570 sec 2024-11-28T01:42:26,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fdc35afc070d49a1963961d2023420b5_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758146292/Put/seqid=0 2024-11-28T01:42:26,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742241_1417 (size=12154) 2024-11-28T01:42:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758206367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758206369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758206384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758206412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758206413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758206492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758206492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758206503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758206633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758206633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758206703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758206703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758206722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,742 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:26,746 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128fdc35afc070d49a1963961d2023420b5_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fdc35afc070d49a1963961d2023420b5_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:26,747 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/be1c0c212fe54e219c217865799f13ae, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:26,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/be1c0c212fe54e219c217865799f13ae is 175, key is test_row_0/A:col10/1732758146292/Put/seqid=0 2024-11-28T01:42:26,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742242_1418 (size=30955) 2024-11-28T01:42:26,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/be1c0c212fe54e219c217865799f13ae 2024-11-28T01:42:26,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/cc80ec0eb91b4a5b9f38a31ac7939ab3 is 50, key is test_row_0/B:col10/1732758146292/Put/seqid=0 2024-11-28T01:42:26,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742243_1419 (size=12001) 2024-11-28T01:42:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T01:42:26,822 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:26,823 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-28T01:42:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T01:42:26,824 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:26,825 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:26,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T01:42:26,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758206952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758206952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:26,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T01:42:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:26,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:26,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758207022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758207022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758207042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T01:42:27,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:27,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T01:42:27,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/cc80ec0eb91b4a5b9f38a31ac7939ab3 2024-11-28T01:42:27,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/4ab14fca3906406fb278363631ba4fcf is 50, key is test_row_0/C:col10/1732758146292/Put/seqid=0 2024-11-28T01:42:27,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742244_1420 (size=12001) 2024-11-28T01:42:27,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T01:42:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T01:42:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T01:42:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758207472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758207472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758207543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758207543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758207562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:27,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/4ab14fca3906406fb278363631ba4fcf 2024-11-28T01:42:27,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T01:42:27,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:27,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:27,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/be1c0c212fe54e219c217865799f13ae as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/be1c0c212fe54e219c217865799f13ae 2024-11-28T01:42:27,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/be1c0c212fe54e219c217865799f13ae, entries=150, sequenceid=119, filesize=30.2 K 2024-11-28T01:42:27,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/cc80ec0eb91b4a5b9f38a31ac7939ab3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/cc80ec0eb91b4a5b9f38a31ac7939ab3 2024-11-28T01:42:27,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/cc80ec0eb91b4a5b9f38a31ac7939ab3, entries=150, sequenceid=119, filesize=11.7 K 2024-11-28T01:42:27,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/4ab14fca3906406fb278363631ba4fcf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4ab14fca3906406fb278363631ba4fcf 2024-11-28T01:42:27,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4ab14fca3906406fb278363631ba4fcf, entries=150, sequenceid=119, filesize=11.7 K 2024-11-28T01:42:27,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132415, heapSize ~364.49 KB/373240, currentSize=53.67 KB/54960 for 9420be97650ba7f85aad07a9b4f2cb6f in 1291ms, sequenceid=119, compaction requested=true 2024-11-28T01:42:27,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:27,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:27,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:27,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:27,612 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:27,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:27,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:27,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:27,612 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:27,613 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:27,613 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:27,613 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:27,613 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:27,613 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,613 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,613 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/c4dc6c8e3d25497f96a80635d76b9778, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d192eabe277147d8bf455458e0aa87ab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/be1c0c212fe54e219c217865799f13ae] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=90.8 K 2024-11-28T01:42:27,613 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/6eb7722538594ab9a099421fed8c5235, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/74fff2bbaae14beb9bec5d9cb15242eb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/cc80ec0eb91b4a5b9f38a31ac7939ab3] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=35.3 K 2024-11-28T01:42:27,613 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,613 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/c4dc6c8e3d25497f96a80635d76b9778, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d192eabe277147d8bf455458e0aa87ab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/be1c0c212fe54e219c217865799f13ae] 2024-11-28T01:42:27,613 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb7722538594ab9a099421fed8c5235, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732758142818 2024-11-28T01:42:27,613 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4dc6c8e3d25497f96a80635d76b9778, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732758142818 2024-11-28T01:42:27,614 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 74fff2bbaae14beb9bec5d9cb15242eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732758144014 2024-11-28T01:42:27,614 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d192eabe277147d8bf455458e0aa87ab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732758144014 2024-11-28T01:42:27,614 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting cc80ec0eb91b4a5b9f38a31ac7939ab3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732758146281 2024-11-28T01:42:27,614 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting be1c0c212fe54e219c217865799f13ae, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732758146281 2024-11-28T01:42:27,624 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:27,627 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:27,627 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411281c12651d5278406699406a9aeddd7dc7_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:27,627 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/54142080464f4252b1cca63648025202 is 50, key is test_row_0/B:col10/1732758146292/Put/seqid=0 2024-11-28T01:42:27,629 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411281c12651d5278406699406a9aeddd7dc7_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:27,629 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281c12651d5278406699406a9aeddd7dc7_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:27,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742246_1422 (size=4469) 2024-11-28T01:42:27,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742245_1421 (size=12241) 2024-11-28T01:42:27,651 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/54142080464f4252b1cca63648025202 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/54142080464f4252b1cca63648025202 2024-11-28T01:42:27,656 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into 54142080464f4252b1cca63648025202(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:27,656 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:27,656 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=13, startTime=1732758147612; duration=0sec 2024-11-28T01:42:27,656 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:27,656 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:27,656 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:27,659 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:27,659 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:27,659 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,659 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/0088d599e4834a93a40045f4c6ddcc7e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/8578c06b05734b259c0d3c313acbf2d7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4ab14fca3906406fb278363631ba4fcf] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=35.3 K 2024-11-28T01:42:27,659 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0088d599e4834a93a40045f4c6ddcc7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732758142818 2024-11-28T01:42:27,659 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8578c06b05734b259c0d3c313acbf2d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732758144014 2024-11-28T01:42:27,660 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ab14fca3906406fb278363631ba4fcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732758146281 2024-11-28T01:42:27,669 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:27,670 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/e6ca2ccbd1504b31aff79c643e3f9ad9 is 50, key is test_row_0/C:col10/1732758146292/Put/seqid=0 2024-11-28T01:42:27,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742247_1423 (size=12241) 2024-11-28T01:42:27,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T01:42:27,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:27,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T01:42:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:27,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128db9d4c46250b4882aa00eb45bab9c9cf_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758146382/Put/seqid=0 2024-11-28T01:42:27,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742248_1424 (size=12204) 2024-11-28T01:42:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T01:42:28,047 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#358 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:28,048 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/6e6603afae6a4a2b9016a90723071695 is 175, key is test_row_0/A:col10/1732758146292/Put/seqid=0 2024-11-28T01:42:28,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742249_1425 (size=31195) 2024-11-28T01:42:28,084 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/e6ca2ccbd1504b31aff79c643e3f9ad9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e6ca2ccbd1504b31aff79c643e3f9ad9 2024-11-28T01:42:28,090 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into e6ca2ccbd1504b31aff79c643e3f9ad9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:28,090 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:28,090 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=13, startTime=1732758147612; duration=0sec 2024-11-28T01:42:28,090 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:28,090 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:28,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,165 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128db9d4c46250b4882aa00eb45bab9c9cf_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128db9d4c46250b4882aa00eb45bab9c9cf_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:28,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/fe320457286e4787a7dc162a382f6bf3, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/fe320457286e4787a7dc162a382f6bf3 is 175, key is test_row_0/A:col10/1732758146382/Put/seqid=0 2024-11-28T01:42:28,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742250_1426 (size=31005) 2024-11-28T01:42:28,171 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/fe320457286e4787a7dc162a382f6bf3 2024-11-28T01:42:28,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/2d9d98a6fd52421198492c00005472d6 is 50, key is test_row_0/B:col10/1732758146382/Put/seqid=0 2024-11-28T01:42:28,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742251_1427 (size=12051) 2024-11-28T01:42:28,203 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/2d9d98a6fd52421198492c00005472d6 2024-11-28T01:42:28,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/dde7ba53bc8f4769ab38d12fbdb6f515 is 50, key is test_row_0/C:col10/1732758146382/Put/seqid=0 2024-11-28T01:42:28,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742252_1428 (size=12051) 2024-11-28T01:42:28,236 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/dde7ba53bc8f4769ab38d12fbdb6f515 2024-11-28T01:42:28,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/fe320457286e4787a7dc162a382f6bf3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/fe320457286e4787a7dc162a382f6bf3 2024-11-28T01:42:28,251 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/fe320457286e4787a7dc162a382f6bf3, entries=150, sequenceid=131, filesize=30.3 K 2024-11-28T01:42:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/2d9d98a6fd52421198492c00005472d6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2d9d98a6fd52421198492c00005472d6 2024-11-28T01:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,256 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2d9d98a6fd52421198492c00005472d6, entries=150, sequenceid=131, filesize=11.8 K 2024-11-28T01:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/dde7ba53bc8f4769ab38d12fbdb6f515 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dde7ba53bc8f4769ab38d12fbdb6f515 2024-11-28T01:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,261 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dde7ba53bc8f4769ab38d12fbdb6f515, entries=150, sequenceid=131, filesize=11.8 K 2024-11-28T01:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,263 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 9420be97650ba7f85aad07a9b4f2cb6f in 518ms, sequenceid=131, compaction requested=false 2024-11-28T01:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-28T01:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-28T01:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-28T01:42:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-11-28T01:42:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4450 sec 2024-11-28T01:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,457 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/6e6603afae6a4a2b9016a90723071695 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e6603afae6a4a2b9016a90723071695 2024-11-28T01:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,464 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into 6e6603afae6a4a2b9016a90723071695(size=30.5 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,464 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:28,464 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=13, startTime=1732758147612; duration=0sec 2024-11-28T01:42:28,464 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:28,464 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:28,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:42:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:28,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284d9087ece76d4de8831d04ff7f213fab_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758148644/Put/seqid=0 2024-11-28T01:42:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742253_1429 (size=19774) 2024-11-28T01:42:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284d9087ece76d4de8831d04ff7f213fab_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284d9087ece76d4de8831d04ff7f213fab_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/87b339f8ffae49a189dd78fd8f223965, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/87b339f8ffae49a189dd78fd8f223965 is 175, key is test_row_0/A:col10/1732758148644/Put/seqid=0 2024-11-28T01:42:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742254_1430 (size=57033) 2024-11-28T01:42:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:28,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758148709, max timestamp=1732758148745Num uniques 859; 2024-11-28T01:42:28,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:42:28,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758148709, max timestamp=1732758148745Num uniques 859; 2024-11-28T01:42:28,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:42:28,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758148709, max timestamp=1732758148745Num uniques 859; 2024-11-28T01:42:28,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:42:28,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758208796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758208795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758208798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758208814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758208814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758208923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758208923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758208923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758208923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758208942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T01:42:28,973 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:28,974 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-28T01:42:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T01:42:28,975 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:28,976 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:28,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:29,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/87b339f8ffae49a189dd78fd8f223965 2024-11-28T01:42:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T01:42:29,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7e5702831b3d4feea9afac6635d3df91 is 50, key is test_row_0/B:col10/1732758148644/Put/seqid=0 2024-11-28T01:42:29,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742255_1431 (size=12151) 2024-11-28T01:42:29,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758209142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758209143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758209143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758209143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758209162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:29,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:29,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T01:42:29,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758209463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758209463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758209464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758209464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758209483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:29,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7e5702831b3d4feea9afac6635d3df91 2024-11-28T01:42:29,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/abc3c47e34c14d058df890d933ae6d51 is 50, key is test_row_0/C:col10/1732758148644/Put/seqid=0 2024-11-28T01:42:29,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742256_1432 (size=12151) 2024-11-28T01:42:29,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:29,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:29,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T01:42:29,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:29,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:29,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:29,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/abc3c47e34c14d058df890d933ae6d51 2024-11-28T01:42:29,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/87b339f8ffae49a189dd78fd8f223965 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/87b339f8ffae49a189dd78fd8f223965 2024-11-28T01:42:29,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/87b339f8ffae49a189dd78fd8f223965, entries=300, sequenceid=143, filesize=55.7 K 2024-11-28T01:42:29,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7e5702831b3d4feea9afac6635d3df91 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7e5702831b3d4feea9afac6635d3df91 2024-11-28T01:42:29,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7e5702831b3d4feea9afac6635d3df91, entries=150, sequenceid=143, filesize=11.9 K 2024-11-28T01:42:29,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/abc3c47e34c14d058df890d933ae6d51 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/abc3c47e34c14d058df890d933ae6d51 2024-11-28T01:42:29,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/abc3c47e34c14d058df890d933ae6d51, entries=150, sequenceid=143, filesize=11.9 K 2024-11-28T01:42:29,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 9420be97650ba7f85aad07a9b4f2cb6f in 1314ms, sequenceid=143, compaction requested=true 2024-11-28T01:42:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:29,960 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:29,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:42:29,960 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:29,969 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:29,970 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:29,970 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,970 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e6603afae6a4a2b9016a90723071695, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/fe320457286e4787a7dc162a382f6bf3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/87b339f8ffae49a189dd78fd8f223965] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=116.4 K 2024-11-28T01:42:29,970 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,970 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e6603afae6a4a2b9016a90723071695, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/fe320457286e4787a7dc162a382f6bf3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/87b339f8ffae49a189dd78fd8f223965] 2024-11-28T01:42:29,971 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:29,972 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6603afae6a4a2b9016a90723071695, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732758146281 2024-11-28T01:42:29,972 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:29,972 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:29,972 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/54142080464f4252b1cca63648025202, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2d9d98a6fd52421198492c00005472d6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7e5702831b3d4feea9afac6635d3df91] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=35.6 K 2024-11-28T01:42:29,977 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 54142080464f4252b1cca63648025202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732758146281 2024-11-28T01:42:29,977 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe320457286e4787a7dc162a382f6bf3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732758146335 2024-11-28T01:42:29,977 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d9d98a6fd52421198492c00005472d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732758146335 2024-11-28T01:42:29,977 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87b339f8ffae49a189dd78fd8f223965, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732758148557 2024-11-28T01:42:29,978 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5702831b3d4feea9afac6635d3df91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732758148643 2024-11-28T01:42:29,995 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:29,997 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:29,997 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/e627c4892bff442593da108c53247898 is 50, key is test_row_0/B:col10/1732758148644/Put/seqid=0 2024-11-28T01:42:30,002 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128b0853521b34e4ba09bc594d80cd1118f_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-28T01:42:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:30,004 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128b0853521b34e4ba09bc594d80cd1118f_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,004 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b0853521b34e4ba09bc594d80cd1118f_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732758148782, max timestamp=1732758150000Num uniques 1068; 2024-11-28T01:42:30,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-28T01:42:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:42:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:42:30,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742257_1433 (size=12493) 2024-11-28T01:42:30,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d5bc0c401f984102bf7e48b47b24c0db_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758148782/Put/seqid=0 2024-11-28T01:42:30,052 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/e627c4892bff442593da108c53247898 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e627c4892bff442593da108c53247898 2024-11-28T01:42:30,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,059 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into e627c4892bff442593da108c53247898(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:30,059 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:30,059 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=13, startTime=1732758149960; duration=0sec 2024-11-28T01:42:30,059 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:30,059 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:30,059 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:30,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742258_1434 (size=4469) 2024-11-28T01:42:30,061 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:30,061 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:30,062 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,062 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e6ca2ccbd1504b31aff79c643e3f9ad9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dde7ba53bc8f4769ab38d12fbdb6f515, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/abc3c47e34c14d058df890d933ae6d51] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=35.6 K 2024-11-28T01:42:30,062 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ca2ccbd1504b31aff79c643e3f9ad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732758146281 2024-11-28T01:42:30,063 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting dde7ba53bc8f4769ab38d12fbdb6f515, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732758146335 2024-11-28T01:42:30,063 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting abc3c47e34c14d058df890d933ae6d51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732758148643 2024-11-28T01:42:30,067 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#368 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:30,067 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/9f69babe0c4b4201b6de60805de73d02 is 175, key is test_row_0/A:col10/1732758148644/Put/seqid=0 2024-11-28T01:42:30,089 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:30,090 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/12a13be6e97f4d238850a8fa5f2e0884 is 50, key is test_row_0/C:col10/1732758148644/Put/seqid=0 2024-11-28T01:42:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758210055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T01:42:30,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758210088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758210090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758210091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758210091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742260_1436 (size=31447) 2024-11-28T01:42:30,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742259_1435 (size=12304) 2024-11-28T01:42:30,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:30,140 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d5bc0c401f984102bf7e48b47b24c0db_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d5bc0c401f984102bf7e48b47b24c0db_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:30,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d769985b7ef4478a8a40ef691e14a9b1, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d769985b7ef4478a8a40ef691e14a9b1 is 175, key is test_row_0/A:col10/1732758148782/Put/seqid=0 2024-11-28T01:42:30,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742261_1437 (size=12493) 2024-11-28T01:42:30,154 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/12a13be6e97f4d238850a8fa5f2e0884 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/12a13be6e97f4d238850a8fa5f2e0884 2024-11-28T01:42:30,162 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into 12a13be6e97f4d238850a8fa5f2e0884(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:30,162 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:30,162 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=13, startTime=1732758149960; duration=0sec 2024-11-28T01:42:30,162 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:30,162 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:30,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742262_1438 (size=31105) 2024-11-28T01:42:30,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d769985b7ef4478a8a40ef691e14a9b1 2024-11-28T01:42:30,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/f742901a4d3d4e969f45195cbacb98e4 is 50, key is test_row_0/B:col10/1732758148782/Put/seqid=0 2024-11-28T01:42:30,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742263_1439 (size=12151) 2024-11-28T01:42:30,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/f742901a4d3d4e969f45195cbacb98e4 2024-11-28T01:42:30,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/d28b6da5abc441c1985d75e9b3368276 is 50, key is test_row_0/C:col10/1732758148782/Put/seqid=0 2024-11-28T01:42:30,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742264_1440 (size=12151) 2024-11-28T01:42:30,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/d28b6da5abc441c1985d75e9b3368276 2024-11-28T01:42:30,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d769985b7ef4478a8a40ef691e14a9b1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d769985b7ef4478a8a40ef691e14a9b1 2024-11-28T01:42:30,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d769985b7ef4478a8a40ef691e14a9b1, entries=150, sequenceid=182, filesize=30.4 K 2024-11-28T01:42:30,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/f742901a4d3d4e969f45195cbacb98e4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/f742901a4d3d4e969f45195cbacb98e4 2024-11-28T01:42:30,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:30,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:30,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/f742901a4d3d4e969f45195cbacb98e4, entries=150, sequenceid=182, filesize=11.9 K 2024-11-28T01:42:30,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/d28b6da5abc441c1985d75e9b3368276 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/d28b6da5abc441c1985d75e9b3368276 2024-11-28T01:42:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/d28b6da5abc441c1985d75e9b3368276, entries=150, sequenceid=182, filesize=11.9 K 2024-11-28T01:42:30,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~327.31 KB/335168, currentSize=67.09 KB/68700 for 9420be97650ba7f85aad07a9b4f2cb6f in 213ms, sequenceid=182, compaction requested=false 2024-11-28T01:42:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:30,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:42:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:30,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411286f8cb0b75c1a4372979bc4408156eb4c_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758150089/Put/seqid=0 2024-11-28T01:42:30,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742265_1441 (size=14794) 2024-11-28T01:42:30,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:30,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411286f8cb0b75c1a4372979bc4408156eb4c_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411286f8cb0b75c1a4372979bc4408156eb4c_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:30,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/eb870154d2dc4180b2598108c2f8a55b, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/eb870154d2dc4180b2598108c2f8a55b is 175, key is test_row_0/A:col10/1732758150089/Put/seqid=0 2024-11-28T01:42:30,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742266_1442 (size=39749) 2024-11-28T01:42:30,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/eb870154d2dc4180b2598108c2f8a55b 2024-11-28T01:42:30,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7de67a2a3e5c45649f12ae60ba32b6f1 is 50, key is test_row_0/B:col10/1732758150089/Put/seqid=0 2024-11-28T01:42:30,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742267_1443 (size=12151) 2024-11-28T01:42:30,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7de67a2a3e5c45649f12ae60ba32b6f1 2024-11-28T01:42:30,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/384c4bd1464a450fa3d1000f9511de11 is 50, key is test_row_0/C:col10/1732758150089/Put/seqid=0 2024-11-28T01:42:30,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742268_1444 (size=12151) 2024-11-28T01:42:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758210269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758210270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758210273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758210285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758210286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:30,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:30,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758210392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758210392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758210393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758210393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758210402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,536 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/9f69babe0c4b4201b6de60805de73d02 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/9f69babe0c4b4201b6de60805de73d02 2024-11-28T01:42:30,541 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into 9f69babe0c4b4201b6de60805de73d02(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:30,541 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:30,541 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=13, startTime=1732758149960; duration=0sec 2024-11-28T01:42:30,541 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:30,541 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:30,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758210603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758210603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758210603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758210603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758210622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:30,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/384c4bd1464a450fa3d1000f9511de11 2024-11-28T01:42:30,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/eb870154d2dc4180b2598108c2f8a55b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/eb870154d2dc4180b2598108c2f8a55b 2024-11-28T01:42:30,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/eb870154d2dc4180b2598108c2f8a55b, entries=200, sequenceid=198, filesize=38.8 K 2024-11-28T01:42:30,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7de67a2a3e5c45649f12ae60ba32b6f1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7de67a2a3e5c45649f12ae60ba32b6f1 2024-11-28T01:42:30,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7de67a2a3e5c45649f12ae60ba32b6f1, entries=150, sequenceid=198, filesize=11.9 K 2024-11-28T01:42:30,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/384c4bd1464a450fa3d1000f9511de11 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/384c4bd1464a450fa3d1000f9511de11 2024-11-28T01:42:30,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/384c4bd1464a450fa3d1000f9511de11, entries=150, sequenceid=198, filesize=11.9 K 2024-11-28T01:42:30,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9420be97650ba7f85aad07a9b4f2cb6f in 481ms, sequenceid=198, compaction requested=true 2024-11-28T01:42:30,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:30,699 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:30,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:30,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:30,699 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:30,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:30,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:30,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:30,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:30,700 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:30,700 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:30,700 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,700 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/9f69babe0c4b4201b6de60805de73d02, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d769985b7ef4478a8a40ef691e14a9b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/eb870154d2dc4180b2598108c2f8a55b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=99.9 K 2024-11-28T01:42:30,700 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,700 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/9f69babe0c4b4201b6de60805de73d02, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d769985b7ef4478a8a40ef691e14a9b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/eb870154d2dc4180b2598108c2f8a55b] 2024-11-28T01:42:30,701 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:30,701 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:30,701 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,701 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e627c4892bff442593da108c53247898, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/f742901a4d3d4e969f45195cbacb98e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7de67a2a3e5c45649f12ae60ba32b6f1] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=35.9 K 2024-11-28T01:42:30,701 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f69babe0c4b4201b6de60805de73d02, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732758148643 2024-11-28T01:42:30,704 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e627c4892bff442593da108c53247898, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732758148643 2024-11-28T01:42:30,704 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d769985b7ef4478a8a40ef691e14a9b1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732758148782 2024-11-28T01:42:30,704 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f742901a4d3d4e969f45195cbacb98e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732758148782 2024-11-28T01:42:30,704 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb870154d2dc4180b2598108c2f8a55b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732758150051 2024-11-28T01:42:30,704 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de67a2a3e5c45649f12ae60ba32b6f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732758150051 2024-11-28T01:42:30,716 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:30,716 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/a3eb9c7331f74b9daa36468dd55cd2c1 is 50, key is test_row_0/B:col10/1732758150089/Put/seqid=0 2024-11-28T01:42:30,719 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742269_1445 (size=12595) 2024-11-28T01:42:30,728 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112804a5372fe4934791a518c57263b63a29_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,731 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112804a5372fe4934791a518c57263b63a29_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,731 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112804a5372fe4934791a518c57263b63a29_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742270_1446 (size=4469) 2024-11-28T01:42:30,760 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#377 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:30,761 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/bdc061c23cca4483a0dd6e02fbeba723 is 175, key is test_row_0/A:col10/1732758150089/Put/seqid=0 2024-11-28T01:42:30,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742271_1447 (size=31549) 2024-11-28T01:42:30,780 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/bdc061c23cca4483a0dd6e02fbeba723 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bdc061c23cca4483a0dd6e02fbeba723 2024-11-28T01:42:30,792 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into bdc061c23cca4483a0dd6e02fbeba723(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:30,792 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:30,792 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=13, startTime=1732758150699; duration=0sec 2024-11-28T01:42:30,793 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:30,793 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:30,793 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:30,794 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:30,794 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:30,794 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,794 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/12a13be6e97f4d238850a8fa5f2e0884, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/d28b6da5abc441c1985d75e9b3368276, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/384c4bd1464a450fa3d1000f9511de11] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=35.9 K 2024-11-28T01:42:30,794 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a13be6e97f4d238850a8fa5f2e0884, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732758148643 2024-11-28T01:42:30,795 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28b6da5abc441c1985d75e9b3368276, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732758148782 2024-11-28T01:42:30,795 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384c4bd1464a450fa3d1000f9511de11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732758150051 2024-11-28T01:42:30,802 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:30,803 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/298dc8be054048c0b880e4069f0582bf is 50, key is test_row_0/C:col10/1732758150089/Put/seqid=0 2024-11-28T01:42:30,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742272_1448 (size=12595) 2024-11-28T01:42:30,819 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/298dc8be054048c0b880e4069f0582bf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/298dc8be054048c0b880e4069f0582bf 2024-11-28T01:42:30,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T01:42:30,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:30,823 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T01:42:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:30,827 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into 298dc8be054048c0b880e4069f0582bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:30,827 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:30,827 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=13, startTime=1732758150700; duration=0sec 2024-11-28T01:42:30,827 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:30,827 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:30,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e3540a19809d4e00b5098fa0858569c3_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758150282/Put/seqid=0 2024-11-28T01:42:30,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742273_1449 (size=12304) 2024-11-28T01:42:30,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:30,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e3540a19809d4e00b5098fa0858569c3_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e3540a19809d4e00b5098fa0858569c3_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/036fb36fd17947119cec983cc2bb5ec4, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/036fb36fd17947119cec983cc2bb5ec4 is 175, key is test_row_0/A:col10/1732758150282/Put/seqid=0 2024-11-28T01:42:30,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742274_1450 (size=31105) 2024-11-28T01:42:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:30,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:30,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758210945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758210945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758210946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758210960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758210960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:30,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758211062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758211062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758211063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758211082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758211082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T01:42:31,135 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/a3eb9c7331f74b9daa36468dd55cd2c1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a3eb9c7331f74b9daa36468dd55cd2c1 2024-11-28T01:42:31,140 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into a3eb9c7331f74b9daa36468dd55cd2c1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:31,140 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:31,140 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=13, startTime=1732758150699; duration=0sec 2024-11-28T01:42:31,140 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:31,140 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:31,271 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/036fb36fd17947119cec983cc2bb5ec4 2024-11-28T01:42:31,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758211273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758211273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758211273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/c83e8b6682cd4566a8b412d3973dbc2a is 50, key is test_row_0/B:col10/1732758150282/Put/seqid=0 2024-11-28T01:42:31,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758211293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742275_1451 (size=12151) 2024-11-28T01:42:31,308 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/c83e8b6682cd4566a8b412d3973dbc2a 2024-11-28T01:42:31,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/ea6494a62b024afdba117e13dbe2bf27 is 50, key is test_row_0/C:col10/1732758150282/Put/seqid=0 2024-11-28T01:42:31,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758211303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742276_1452 (size=12151) 2024-11-28T01:42:31,332 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/ea6494a62b024afdba117e13dbe2bf27 2024-11-28T01:42:31,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/036fb36fd17947119cec983cc2bb5ec4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/036fb36fd17947119cec983cc2bb5ec4 2024-11-28T01:42:31,345 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/036fb36fd17947119cec983cc2bb5ec4, entries=150, sequenceid=218, filesize=30.4 K 2024-11-28T01:42:31,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/c83e8b6682cd4566a8b412d3973dbc2a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c83e8b6682cd4566a8b412d3973dbc2a 2024-11-28T01:42:31,353 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c83e8b6682cd4566a8b412d3973dbc2a, entries=150, sequenceid=218, filesize=11.9 K 2024-11-28T01:42:31,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/ea6494a62b024afdba117e13dbe2bf27 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/ea6494a62b024afdba117e13dbe2bf27 2024-11-28T01:42:31,364 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/ea6494a62b024afdba117e13dbe2bf27, entries=150, sequenceid=218, filesize=11.9 K 2024-11-28T01:42:31,366 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 9420be97650ba7f85aad07a9b4f2cb6f in 543ms, sequenceid=218, compaction requested=false 2024-11-28T01:42:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-28T01:42:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-28T01:42:31,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-28T01:42:31,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3910 sec 2024-11-28T01:42:31,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.3950 sec 2024-11-28T01:42:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:31,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T01:42:31,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:31,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:31,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:31,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:31,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:31,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:31,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112885cf8b56a20f4a1a9403d541741093a7_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758151587/Put/seqid=0 2024-11-28T01:42:31,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742277_1453 (size=14794) 2024-11-28T01:42:31,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:31,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112885cf8b56a20f4a1a9403d541741093a7_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112885cf8b56a20f4a1a9403d541741093a7_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:31,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b32612da26ec4dd2a92af8d914760bcc, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:31,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b32612da26ec4dd2a92af8d914760bcc is 175, key is test_row_0/A:col10/1732758151587/Put/seqid=0 2024-11-28T01:42:31,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742278_1454 (size=39749) 2024-11-28T01:42:31,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b32612da26ec4dd2a92af8d914760bcc 2024-11-28T01:42:31,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758211627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/752edd99cc69460b928c506eb241ec90 is 50, key is test_row_0/B:col10/1732758151587/Put/seqid=0 2024-11-28T01:42:31,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758211639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758211641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758211645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758211646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742279_1455 (size=12151) 2024-11-28T01:42:31,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/752edd99cc69460b928c506eb241ec90 2024-11-28T01:42:31,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/14bf2f695621495183e50d957ad9d3a6 is 50, key is test_row_0/C:col10/1732758151587/Put/seqid=0 2024-11-28T01:42:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742280_1456 (size=12151) 2024-11-28T01:42:31,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/14bf2f695621495183e50d957ad9d3a6 2024-11-28T01:42:31,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b32612da26ec4dd2a92af8d914760bcc as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b32612da26ec4dd2a92af8d914760bcc 2024-11-28T01:42:31,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b32612da26ec4dd2a92af8d914760bcc, entries=200, sequenceid=236, filesize=38.8 K 2024-11-28T01:42:31,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/752edd99cc69460b928c506eb241ec90 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/752edd99cc69460b928c506eb241ec90 2024-11-28T01:42:31,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/752edd99cc69460b928c506eb241ec90, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T01:42:31,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/14bf2f695621495183e50d957ad9d3a6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/14bf2f695621495183e50d957ad9d3a6 2024-11-28T01:42:31,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/14bf2f695621495183e50d957ad9d3a6, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T01:42:31,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 9420be97650ba7f85aad07a9b4f2cb6f in 136ms, sequenceid=236, compaction requested=true 2024-11-28T01:42:31,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:31,725 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:31,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:31,726 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:31,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:31,726 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:31,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:31,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:31,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:31,727 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:31,727 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:31,727 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bdc061c23cca4483a0dd6e02fbeba723, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/036fb36fd17947119cec983cc2bb5ec4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b32612da26ec4dd2a92af8d914760bcc] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=100.0 K 2024-11-28T01:42:31,727 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:31,727 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bdc061c23cca4483a0dd6e02fbeba723, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/036fb36fd17947119cec983cc2bb5ec4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b32612da26ec4dd2a92af8d914760bcc] 2024-11-28T01:42:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:31,728 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdc061c23cca4483a0dd6e02fbeba723, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732758150051 2024-11-28T01:42:31,729 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036fb36fd17947119cec983cc2bb5ec4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732758150261 2024-11-28T01:42:31,729 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:31,729 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:31,729 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:31,729 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a3eb9c7331f74b9daa36468dd55cd2c1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c83e8b6682cd4566a8b412d3973dbc2a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/752edd99cc69460b928c506eb241ec90] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=36.0 K 2024-11-28T01:42:31,730 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b32612da26ec4dd2a92af8d914760bcc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758150941 2024-11-28T01:42:31,730 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a3eb9c7331f74b9daa36468dd55cd2c1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732758150051 2024-11-28T01:42:31,730 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c83e8b6682cd4566a8b412d3973dbc2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732758150261 2024-11-28T01:42:31,731 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 752edd99cc69460b928c506eb241ec90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758150941 2024-11-28T01:42:31,740 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:31,746 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:31,747 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/a47a69167aec4131bfcddaf6ce4163e8 is 50, key is test_row_0/B:col10/1732758151587/Put/seqid=0 2024-11-28T01:42:31,751 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128a7f0a1c5f3744e73b97f36da17050511_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:31,754 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128a7f0a1c5f3744e73b97f36da17050511_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:31,754 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a7f0a1c5f3744e73b97f36da17050511_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:31,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T01:42:31,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:31,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:31,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:31,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742281_1457 (size=12697) 2024-11-28T01:42:31,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742282_1458 (size=4469) 2024-11-28T01:42:31,779 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#385 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:31,780 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/261087f33d2c4df09905da37f4eb7373 is 175, key is test_row_0/A:col10/1732758151587/Put/seqid=0 2024-11-28T01:42:31,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f340c6b8a88d467fae120f6d29bead3b_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758151642/Put/seqid=0 2024-11-28T01:42:31,787 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/a47a69167aec4131bfcddaf6ce4163e8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a47a69167aec4131bfcddaf6ce4163e8 2024-11-28T01:42:31,797 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into a47a69167aec4131bfcddaf6ce4163e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:31,797 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:31,797 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=13, startTime=1732758151726; duration=0sec 2024-11-28T01:42:31,797 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:31,797 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:31,797 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:31,798 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:31,798 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:31,798 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:31,798 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/298dc8be054048c0b880e4069f0582bf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/ea6494a62b024afdba117e13dbe2bf27, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/14bf2f695621495183e50d957ad9d3a6] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=36.0 K 2024-11-28T01:42:31,799 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 298dc8be054048c0b880e4069f0582bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732758150051 2024-11-28T01:42:31,799 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6494a62b024afdba117e13dbe2bf27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732758150261 2024-11-28T01:42:31,799 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 14bf2f695621495183e50d957ad9d3a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758150941 2024-11-28T01:42:31,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742283_1459 (size=31651) 2024-11-28T01:42:31,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742284_1460 (size=14794) 2024-11-28T01:42:31,826 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:31,827 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/619ec4647e1b49c5984b15cf8ea2a31c is 50, key is test_row_0/C:col10/1732758151587/Put/seqid=0 2024-11-28T01:42:31,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758211807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758211808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758211808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758211809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758211831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742285_1461 (size=12697) 2024-11-28T01:42:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758211932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758211932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758211932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758211943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758211953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758212153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758212154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758212154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758212162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758212162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,210 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/261087f33d2c4df09905da37f4eb7373 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/261087f33d2c4df09905da37f4eb7373 2024-11-28T01:42:32,217 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into 261087f33d2c4df09905da37f4eb7373(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:32,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:32,217 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=13, startTime=1732758151725; duration=0sec 2024-11-28T01:42:32,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:32,217 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:32,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:32,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f340c6b8a88d467fae120f6d29bead3b_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f340c6b8a88d467fae120f6d29bead3b_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:32,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/a8c30ea47e814e10b41da32beeea67c5, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:32,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/a8c30ea47e814e10b41da32beeea67c5 is 175, key is test_row_0/A:col10/1732758151642/Put/seqid=0 2024-11-28T01:42:32,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742286_1462 (size=39749) 2024-11-28T01:42:32,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/a8c30ea47e814e10b41da32beeea67c5 2024-11-28T01:42:32,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/dc7b5980d5584d8a9d9f0bc2ce1806e3 is 50, key is test_row_0/B:col10/1732758151642/Put/seqid=0 2024-11-28T01:42:32,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742287_1463 (size=12151) 2024-11-28T01:42:32,250 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/619ec4647e1b49c5984b15cf8ea2a31c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/619ec4647e1b49c5984b15cf8ea2a31c 2024-11-28T01:42:32,254 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into 619ec4647e1b49c5984b15cf8ea2a31c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:32,254 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:32,254 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=13, startTime=1732758151727; duration=0sec 2024-11-28T01:42:32,254 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:32,254 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758212473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758212473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758212473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758212482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758212482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:32,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/dc7b5980d5584d8a9d9f0bc2ce1806e3 2024-11-28T01:42:32,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/9adbcefcd4784950b90c2a5fb9741e22 is 50, key is test_row_0/C:col10/1732758151642/Put/seqid=0 2024-11-28T01:42:32,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742288_1464 (size=12151) 2024-11-28T01:42:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758212982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:33,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758212992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:33,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758212992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758212993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:33,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758213003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:33,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/9adbcefcd4784950b90c2a5fb9741e22 2024-11-28T01:42:33,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/a8c30ea47e814e10b41da32beeea67c5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/a8c30ea47e814e10b41da32beeea67c5 2024-11-28T01:42:33,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/a8c30ea47e814e10b41da32beeea67c5, entries=200, sequenceid=254, filesize=38.8 K 2024-11-28T01:42:33,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/dc7b5980d5584d8a9d9f0bc2ce1806e3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/dc7b5980d5584d8a9d9f0bc2ce1806e3 2024-11-28T01:42:33,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/dc7b5980d5584d8a9d9f0bc2ce1806e3, entries=150, sequenceid=254, filesize=11.9 K 2024-11-28T01:42:33,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/9adbcefcd4784950b90c2a5fb9741e22 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9adbcefcd4784950b90c2a5fb9741e22 2024-11-28T01:42:33,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9adbcefcd4784950b90c2a5fb9741e22, entries=150, sequenceid=254, filesize=11.9 K 2024-11-28T01:42:33,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9420be97650ba7f85aad07a9b4f2cb6f in 1322ms, sequenceid=254, compaction requested=false 2024-11-28T01:42:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T01:42:33,143 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:33,144 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-28T01:42:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T01:42:33,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:33,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:33,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:33,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T01:42:33,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T01:42:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:33,299 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:42:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:33,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:33,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128201a61306f9543ae98716c243424c774_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758151805/Put/seqid=0 2024-11-28T01:42:33,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742289_1465 (size=12454) 2024-11-28T01:42:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T01:42:33,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:33,717 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128201a61306f9543ae98716c243424c774_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128201a61306f9543ae98716c243424c774_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:33,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/f37979cdfed949fc94e4a8cd34772e9f, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/f37979cdfed949fc94e4a8cd34772e9f is 175, key is test_row_0/A:col10/1732758151805/Put/seqid=0 2024-11-28T01:42:33,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742290_1466 (size=31255) 2024-11-28T01:42:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T01:42:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:34,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:34,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758214040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758214041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758214048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758214050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758214051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,123 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/f37979cdfed949fc94e4a8cd34772e9f 2024-11-28T01:42:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/578eeb0b2fcf4f8d8fac747609cea4cd is 50, key is test_row_0/B:col10/1732758151805/Put/seqid=0 2024-11-28T01:42:34,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742291_1467 (size=12301) 2024-11-28T01:42:34,135 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/578eeb0b2fcf4f8d8fac747609cea4cd 2024-11-28T01:42:34,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/3caee540f7184737a4029ae4c67bd9d2 is 50, key is test_row_0/C:col10/1732758151805/Put/seqid=0 2024-11-28T01:42:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742292_1468 (size=12301) 2024-11-28T01:42:34,154 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/3caee540f7184737a4029ae4c67bd9d2 2024-11-28T01:42:34,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/f37979cdfed949fc94e4a8cd34772e9f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/f37979cdfed949fc94e4a8cd34772e9f 2024-11-28T01:42:34,167 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/f37979cdfed949fc94e4a8cd34772e9f, entries=150, sequenceid=271, filesize=30.5 K 2024-11-28T01:42:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/578eeb0b2fcf4f8d8fac747609cea4cd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/578eeb0b2fcf4f8d8fac747609cea4cd 2024-11-28T01:42:34,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758214163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758214163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758214163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758214163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,173 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/578eeb0b2fcf4f8d8fac747609cea4cd, entries=150, sequenceid=271, filesize=12.0 K 2024-11-28T01:42:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/3caee540f7184737a4029ae4c67bd9d2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3caee540f7184737a4029ae4c67bd9d2 2024-11-28T01:42:34,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3caee540f7184737a4029ae4c67bd9d2, entries=150, sequenceid=271, filesize=12.0 K 2024-11-28T01:42:34,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 9420be97650ba7f85aad07a9b4f2cb6f in 878ms, sequenceid=271, compaction requested=true 2024-11-28T01:42:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-28T01:42:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-28T01:42:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:34,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:42:34,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:34,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:34,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:34,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:34,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:34,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:34,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-28T01:42:34,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-28T01:42:34,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0430 sec 2024-11-28T01:42:34,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128124e0660c4414b13a7f6744f08f18f2b_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758154035/Put/seqid=0 2024-11-28T01:42:34,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742293_1469 (size=14994) 2024-11-28T01:42:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758214262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T01:42:34,283 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:34,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-28T01:42:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T01:42:34,286 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:34,286 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:34,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T01:42:34,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758214383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758214383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758214383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758214383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758214383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T01:42:34,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T01:42:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T01:42:34,609 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128124e0660c4414b13a7f6744f08f18f2b_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128124e0660c4414b13a7f6744f08f18f2b_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:34,610 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d9ae81aee3ef4095b7da6600dd37c66b, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:34,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d9ae81aee3ef4095b7da6600dd37c66b is 175, key is test_row_0/A:col10/1732758154035/Put/seqid=0 2024-11-28T01:42:34,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742294_1470 (size=39949) 2024-11-28T01:42:34,615 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d9ae81aee3ef4095b7da6600dd37c66b 2024-11-28T01:42:34,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758214603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/2bff0f4b9dc34f988679e133162fab1f is 50, key is test_row_0/B:col10/1732758154035/Put/seqid=0 2024-11-28T01:42:34,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742295_1471 (size=12301) 2024-11-28T01:42:34,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/2bff0f4b9dc34f988679e133162fab1f 2024-11-28T01:42:34,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/1bc73086d06342fa8dc5f574c8887099 is 50, key is test_row_0/C:col10/1732758154035/Put/seqid=0 2024-11-28T01:42:34,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742296_1472 (size=12301) 2024-11-28T01:42:34,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758214702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758214702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758214702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758214702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:34,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T01:42:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T01:42:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:34,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T01:42:34,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758214922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/1bc73086d06342fa8dc5f574c8887099 2024-11-28T01:42:35,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T01:42:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:35,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:35,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/d9ae81aee3ef4095b7da6600dd37c66b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d9ae81aee3ef4095b7da6600dd37c66b 2024-11-28T01:42:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:35,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d9ae81aee3ef4095b7da6600dd37c66b, entries=200, sequenceid=291, filesize=39.0 K 2024-11-28T01:42:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/2bff0f4b9dc34f988679e133162fab1f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2bff0f4b9dc34f988679e133162fab1f 2024-11-28T01:42:35,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2bff0f4b9dc34f988679e133162fab1f, entries=150, sequenceid=291, filesize=12.0 K 2024-11-28T01:42:35,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/1bc73086d06342fa8dc5f574c8887099 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/1bc73086d06342fa8dc5f574c8887099 2024-11-28T01:42:35,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/1bc73086d06342fa8dc5f574c8887099, entries=150, sequenceid=291, filesize=12.0 K 2024-11-28T01:42:35,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9420be97650ba7f85aad07a9b4f2cb6f in 885ms, sequenceid=291, compaction requested=true 2024-11-28T01:42:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:35,069 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:35,069 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:35,071 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:35,071 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:35,071 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:35,071 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:35,071 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:35,071 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/261087f33d2c4df09905da37f4eb7373, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/a8c30ea47e814e10b41da32beeea67c5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/f37979cdfed949fc94e4a8cd34772e9f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d9ae81aee3ef4095b7da6600dd37c66b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=139.3 K 2024-11-28T01:42:35,071 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:35,071 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:35,071 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/261087f33d2c4df09905da37f4eb7373, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/a8c30ea47e814e10b41da32beeea67c5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/f37979cdfed949fc94e4a8cd34772e9f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d9ae81aee3ef4095b7da6600dd37c66b] 2024-11-28T01:42:35,071 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a47a69167aec4131bfcddaf6ce4163e8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/dc7b5980d5584d8a9d9f0bc2ce1806e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/578eeb0b2fcf4f8d8fac747609cea4cd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2bff0f4b9dc34f988679e133162fab1f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=48.3 K 2024-11-28T01:42:35,071 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a47a69167aec4131bfcddaf6ce4163e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758150941 2024-11-28T01:42:35,071 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261087f33d2c4df09905da37f4eb7373, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758150941 2024-11-28T01:42:35,072 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7b5980d5584d8a9d9f0bc2ce1806e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732758151639 2024-11-28T01:42:35,072 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c30ea47e814e10b41da32beeea67c5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732758151639 2024-11-28T01:42:35,072 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 578eeb0b2fcf4f8d8fac747609cea4cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732758151805 2024-11-28T01:42:35,072 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f37979cdfed949fc94e4a8cd34772e9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732758151805 2024-11-28T01:42:35,072 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bff0f4b9dc34f988679e133162fab1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732758154035 2024-11-28T01:42:35,072 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9ae81aee3ef4095b7da6600dd37c66b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732758154035 2024-11-28T01:42:35,079 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:35,080 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:35,081 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/4b80927090ea41dca8f687c1bfa301ff is 50, key is test_row_0/B:col10/1732758154035/Put/seqid=0 2024-11-28T01:42:35,081 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128b11d0c2856a5423ba112b84eef708e0a_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:35,084 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128b11d0c2856a5423ba112b84eef708e0a_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:35,084 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b11d0c2856a5423ba112b84eef708e0a_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:35,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742297_1473 (size=12983) 2024-11-28T01:42:35,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742298_1474 (size=4469) 2024-11-28T01:42:35,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T01:42:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:35,207 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:42:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:35,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:35,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112806aef69b37f24dd7957231bcc3595333_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758154252/Put/seqid=0 2024-11-28T01:42:35,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742299_1475 (size=12454) 2024-11-28T01:42:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:35,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:35,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758215270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758215271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758215277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758215286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758215393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758215393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758215393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758215402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T01:42:35,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758215442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,497 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/4b80927090ea41dca8f687c1bfa301ff as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/4b80927090ea41dca8f687c1bfa301ff 2024-11-28T01:42:35,501 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#398 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:35,501 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/6e732e08bca74daf9cb442bbc758bbcd is 175, key is test_row_0/A:col10/1732758154035/Put/seqid=0 2024-11-28T01:42:35,504 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into 4b80927090ea41dca8f687c1bfa301ff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:35,504 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:35,504 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=12, startTime=1732758155069; duration=0sec 2024-11-28T01:42:35,504 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:35,504 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:35,504 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:35,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742300_1476 (size=31937) 2024-11-28T01:42:35,505 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:35,506 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:35,506 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:35,506 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/619ec4647e1b49c5984b15cf8ea2a31c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9adbcefcd4784950b90c2a5fb9741e22, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3caee540f7184737a4029ae4c67bd9d2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/1bc73086d06342fa8dc5f574c8887099] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=48.3 K 2024-11-28T01:42:35,506 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 619ec4647e1b49c5984b15cf8ea2a31c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758150941 2024-11-28T01:42:35,506 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9adbcefcd4784950b90c2a5fb9741e22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732758151639 2024-11-28T01:42:35,506 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3caee540f7184737a4029ae4c67bd9d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732758151805 2024-11-28T01:42:35,507 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc73086d06342fa8dc5f574c8887099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732758154035 2024-11-28T01:42:35,517 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:35,518 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/08e1ddc38a4b4d4bb659483be99fd47e is 50, key is test_row_0/C:col10/1732758154035/Put/seqid=0 2024-11-28T01:42:35,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742301_1477 (size=12983) 2024-11-28T01:42:35,525 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/08e1ddc38a4b4d4bb659483be99fd47e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/08e1ddc38a4b4d4bb659483be99fd47e 2024-11-28T01:42:35,529 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into 08e1ddc38a4b4d4bb659483be99fd47e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:35,529 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:35,529 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=12, startTime=1732758155069; duration=0sec 2024-11-28T01:42:35,529 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:35,529 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:35,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758215612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:35,627 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112806aef69b37f24dd7957231bcc3595333_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112806aef69b37f24dd7957231bcc3595333_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:35,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1f6327150a0b437d86084bf49046c6b2, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:35,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1f6327150a0b437d86084bf49046c6b2 is 175, key is test_row_0/A:col10/1732758154252/Put/seqid=0 2024-11-28T01:42:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758215622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758215623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758215623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742302_1478 (size=31255) 2024-11-28T01:42:35,910 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/6e732e08bca74daf9cb442bbc758bbcd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e732e08bca74daf9cb442bbc758bbcd 2024-11-28T01:42:35,915 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into 6e732e08bca74daf9cb442bbc758bbcd(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:35,915 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:35,915 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=12, startTime=1732758155069; duration=0sec 2024-11-28T01:42:35,915 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:35,915 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:35,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758215922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758215942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758215942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758215942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:36,036 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1f6327150a0b437d86084bf49046c6b2 2024-11-28T01:42:36,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/30869f82eeb1431c87484cae817f2a24 is 50, key is test_row_0/B:col10/1732758154252/Put/seqid=0 2024-11-28T01:42:36,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742303_1479 (size=12301) 2024-11-28T01:42:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T01:42:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758216442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:36,447 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/30869f82eeb1431c87484cae817f2a24 2024-11-28T01:42:36,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/4b79161cc3e443d0a3d6771641029462 is 50, key is test_row_0/C:col10/1732758154252/Put/seqid=0 2024-11-28T01:42:36,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742304_1480 (size=12301) 2024-11-28T01:42:36,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758216463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:36,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758216463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:36,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758216463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:36,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758216463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:36,470 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 9420be97650ba7f85aad07a9b4f2cb6f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:36,862 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/4b79161cc3e443d0a3d6771641029462 2024-11-28T01:42:36,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/1f6327150a0b437d86084bf49046c6b2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1f6327150a0b437d86084bf49046c6b2 2024-11-28T01:42:36,870 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1f6327150a0b437d86084bf49046c6b2, entries=150, sequenceid=303, filesize=30.5 K 2024-11-28T01:42:36,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/30869f82eeb1431c87484cae817f2a24 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/30869f82eeb1431c87484cae817f2a24 2024-11-28T01:42:36,878 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/30869f82eeb1431c87484cae817f2a24, entries=150, sequenceid=303, filesize=12.0 K 2024-11-28T01:42:36,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/4b79161cc3e443d0a3d6771641029462 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4b79161cc3e443d0a3d6771641029462 2024-11-28T01:42:36,882 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4b79161cc3e443d0a3d6771641029462, entries=150, sequenceid=303, filesize=12.0 K 2024-11-28T01:42:36,883 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9420be97650ba7f85aad07a9b4f2cb6f in 1676ms, sequenceid=303, compaction requested=false 2024-11-28T01:42:36,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:36,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:36,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-28T01:42:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-28T01:42:36,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-28T01:42:36,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5980 sec 2024-11-28T01:42:36,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6040 sec 2024-11-28T01:42:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:37,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T01:42:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:37,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:37,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128498192fe83144126ae61a8909177559f_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758157457/Put/seqid=0 2024-11-28T01:42:37,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742305_1481 (size=14994) 2024-11-28T01:42:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758217491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758217492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758217493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758217495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758217603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758217622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758217622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758217622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758217823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758217842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758217842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758217842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:37,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:37,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128498192fe83144126ae61a8909177559f_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128498192fe83144126ae61a8909177559f_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:37,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/5ebaa00b30474332acc56913e361b801, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:37,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/5ebaa00b30474332acc56913e361b801 is 175, key is test_row_0/A:col10/1732758157457/Put/seqid=0 2024-11-28T01:42:37,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742306_1482 (size=39949) 2024-11-28T01:42:37,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/5ebaa00b30474332acc56913e361b801 2024-11-28T01:42:37,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/50de29f606bc49ce95704c67f208db78 is 50, key is test_row_0/B:col10/1732758157457/Put/seqid=0 2024-11-28T01:42:37,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742307_1483 (size=12301) 2024-11-28T01:42:37,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/50de29f606bc49ce95704c67f208db78 2024-11-28T01:42:37,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a9851025c4b347c7baf73f31b7d8f6ac is 50, key is test_row_0/C:col10/1732758157457/Put/seqid=0 2024-11-28T01:42:38,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742308_1484 (size=12301) 2024-11-28T01:42:38,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a9851025c4b347c7baf73f31b7d8f6ac 2024-11-28T01:42:38,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/5ebaa00b30474332acc56913e361b801 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/5ebaa00b30474332acc56913e361b801 2024-11-28T01:42:38,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/5ebaa00b30474332acc56913e361b801, entries=200, sequenceid=327, filesize=39.0 K 2024-11-28T01:42:38,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/50de29f606bc49ce95704c67f208db78 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/50de29f606bc49ce95704c67f208db78 2024-11-28T01:42:38,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/50de29f606bc49ce95704c67f208db78, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T01:42:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a9851025c4b347c7baf73f31b7d8f6ac as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a9851025c4b347c7baf73f31b7d8f6ac 2024-11-28T01:42:38,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a9851025c4b347c7baf73f31b7d8f6ac, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T01:42:38,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 9420be97650ba7f85aad07a9b4f2cb6f in 578ms, sequenceid=327, compaction requested=true 2024-11-28T01:42:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:38,037 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:38,037 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:38,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:38,038 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:38,038 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:38,039 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,039 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/4b80927090ea41dca8f687c1bfa301ff, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/30869f82eeb1431c87484cae817f2a24, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/50de29f606bc49ce95704c67f208db78] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=36.7 K 2024-11-28T01:42:38,039 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:38,039 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:38,039 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,039 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e732e08bca74daf9cb442bbc758bbcd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1f6327150a0b437d86084bf49046c6b2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/5ebaa00b30474332acc56913e361b801] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=100.7 K 2024-11-28T01:42:38,039 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,039 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e732e08bca74daf9cb442bbc758bbcd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1f6327150a0b437d86084bf49046c6b2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/5ebaa00b30474332acc56913e361b801] 2024-11-28T01:42:38,039 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b80927090ea41dca8f687c1bfa301ff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732758154035 2024-11-28T01:42:38,040 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e732e08bca74daf9cb442bbc758bbcd, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732758154035 2024-11-28T01:42:38,040 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 30869f82eeb1431c87484cae817f2a24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732758154225 2024-11-28T01:42:38,040 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f6327150a0b437d86084bf49046c6b2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732758154225 2024-11-28T01:42:38,040 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 50de29f606bc49ce95704c67f208db78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732758155248 2024-11-28T01:42:38,041 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ebaa00b30474332acc56913e361b801, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732758155248 2024-11-28T01:42:38,060 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:38,066 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:38,067 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/e4478bd170a64e8d97d9aafa8494347c is 50, key is test_row_0/B:col10/1732758157457/Put/seqid=0 2024-11-28T01:42:38,071 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128208fd924600e470ba0fbb86fc980e792_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:38,074 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128208fd924600e470ba0fbb86fc980e792_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:38,074 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128208fd924600e470ba0fbb86fc980e792_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:38,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742309_1485 (size=13085) 2024-11-28T01:42:38,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:42:38,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:38,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:38,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:38,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:38,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:38,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:38,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742310_1486 (size=4469) 2024-11-28T01:42:38,167 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#406 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:38,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/2d827795fc5346d6964c642349c57d5d is 175, key is test_row_0/A:col10/1732758157457/Put/seqid=0 2024-11-28T01:42:38,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288b2eb6d6098448d9a9f0607417cf91e9_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758158147/Put/seqid=0 2024-11-28T01:42:38,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742312_1488 (size=14994) 2024-11-28T01:42:38,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:38,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288b2eb6d6098448d9a9f0607417cf91e9_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288b2eb6d6098448d9a9f0607417cf91e9_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:38,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/de80fd2dafc64a958a10ab07a253211b, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:38,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/de80fd2dafc64a958a10ab07a253211b is 175, key is test_row_0/A:col10/1732758158147/Put/seqid=0 2024-11-28T01:42:38,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742311_1487 (size=32039) 2024-11-28T01:42:38,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758218231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758218233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758218240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742313_1489 (size=39949) 2024-11-28T01:42:38,260 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/de80fd2dafc64a958a10ab07a253211b 2024-11-28T01:42:38,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758218247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7236400131fc4157ad3f01164df7e7a9 is 50, key is test_row_0/B:col10/1732758158147/Put/seqid=0 2024-11-28T01:42:38,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742314_1490 (size=12301) 2024-11-28T01:42:38,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758218352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758218363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758218363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758218373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T01:42:38,453 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:38,454 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-28T01:42:38,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T01:42:38,457 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:38,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:38,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52806 deadline: 1732758218492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,505 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in 9420be97650ba7f85aad07a9b4f2cb6f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:42:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,543 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/e4478bd170a64e8d97d9aafa8494347c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e4478bd170a64e8d97d9aafa8494347c 2024-11-28T01:42:38,551 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into e4478bd170a64e8d97d9aafa8494347c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:38,552 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:38,552 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=13, startTime=1732758158037; duration=0sec 2024-11-28T01:42:38,552 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:38,552 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:38,552 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:38,553 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:38,553 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:38,553 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,553 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/08e1ddc38a4b4d4bb659483be99fd47e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4b79161cc3e443d0a3d6771641029462, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a9851025c4b347c7baf73f31b7d8f6ac] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=36.7 K 2024-11-28T01:42:38,553 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e1ddc38a4b4d4bb659483be99fd47e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732758154035 2024-11-28T01:42:38,554 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b79161cc3e443d0a3d6771641029462, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732758154225 2024-11-28T01:42:38,554 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a9851025c4b347c7baf73f31b7d8f6ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732758155248 2024-11-28T01:42:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T01:42:38,567 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:38,568 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/e80719074e854516aaac916e558ada4c is 50, key is test_row_0/C:col10/1732758157457/Put/seqid=0 2024-11-28T01:42:38,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758218582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T01:42:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758218593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758218593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758218605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742315_1491 (size=13085) 2024-11-28T01:42:38,641 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/2d827795fc5346d6964c642349c57d5d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2d827795fc5346d6964c642349c57d5d 2024-11-28T01:42:38,646 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into 2d827795fc5346d6964c642349c57d5d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:38,647 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:38,647 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=13, startTime=1732758158037; duration=0sec 2024-11-28T01:42:38,647 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:38,647 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:38,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7236400131fc4157ad3f01164df7e7a9 2024-11-28T01:42:38,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/e102fa0fc63840d2a3347fe66ff70fa4 is 50, key is test_row_0/C:col10/1732758158147/Put/seqid=0 2024-11-28T01:42:38,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T01:42:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T01:42:38,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742316_1492 (size=12301) 2024-11-28T01:42:38,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T01:42:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:38,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:38,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758218912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758218923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758218923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:38,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758218932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,045 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/e80719074e854516aaac916e558ada4c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e80719074e854516aaac916e558ada4c 2024-11-28T01:42:39,050 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into e80719074e854516aaac916e558ada4c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:39,050 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:39,050 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=13, startTime=1732758158037; duration=0sec 2024-11-28T01:42:39,050 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:39,050 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:39,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T01:42:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:39,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T01:42:39,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/e102fa0fc63840d2a3347fe66ff70fa4 2024-11-28T01:42:39,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/de80fd2dafc64a958a10ab07a253211b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/de80fd2dafc64a958a10ab07a253211b 2024-11-28T01:42:39,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/de80fd2dafc64a958a10ab07a253211b, entries=200, sequenceid=339, filesize=39.0 K 2024-11-28T01:42:39,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7236400131fc4157ad3f01164df7e7a9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7236400131fc4157ad3f01164df7e7a9 2024-11-28T01:42:39,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7236400131fc4157ad3f01164df7e7a9, entries=150, sequenceid=339, filesize=12.0 K 2024-11-28T01:42:39,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/e102fa0fc63840d2a3347fe66ff70fa4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e102fa0fc63840d2a3347fe66ff70fa4 2024-11-28T01:42:39,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e102fa0fc63840d2a3347fe66ff70fa4, entries=150, sequenceid=339, filesize=12.0 K 2024-11-28T01:42:39,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T01:42:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:39,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9420be97650ba7f85aad07a9b4f2cb6f in 1080ms, sequenceid=339, compaction requested=false 2024-11-28T01:42:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:39,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:39,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:39,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T01:42:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:39,386 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:42:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:39,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:39,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bcb26c05f3d444a4b5c14d70e4e89d54_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758158231/Put/seqid=0 2024-11-28T01:42:39,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742317_1493 (size=12454) 2024-11-28T01:42:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:39,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. as already flushing 2024-11-28T01:42:39,475 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bcb26c05f3d444a4b5c14d70e4e89d54_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bcb26c05f3d444a4b5c14d70e4e89d54_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:39,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/8a26cef0cd24479ab40c7af7f1c979d3, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/8a26cef0cd24479ab40c7af7f1c979d3 is 175, key is test_row_0/A:col10/1732758158231/Put/seqid=0 2024-11-28T01:42:39,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758219499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742318_1494 (size=31255) 2024-11-28T01:42:39,515 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/8a26cef0cd24479ab40c7af7f1c979d3 2024-11-28T01:42:39,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758219500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758219504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758219506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/502b0e7568d4424da3b950d37f5b3f48 is 50, key is test_row_0/B:col10/1732758158231/Put/seqid=0 2024-11-28T01:42:39,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742319_1495 (size=12301) 2024-11-28T01:42:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T01:42:39,595 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/502b0e7568d4424da3b950d37f5b3f48 2024-11-28T01:42:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/85bfd0cac5db45a9b0b839bd8aacbb55 is 50, key is test_row_0/C:col10/1732758158231/Put/seqid=0 2024-11-28T01:42:39,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758219613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742320_1496 (size=12301) 2024-11-28T01:42:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758219632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758219632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758219632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732758219824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52794 deadline: 1732758219853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758219853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52784 deadline: 1732758219854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:40,026 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/85bfd0cac5db45a9b0b839bd8aacbb55 2024-11-28T01:42:40,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/8a26cef0cd24479ab40c7af7f1c979d3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/8a26cef0cd24479ab40c7af7f1c979d3 2024-11-28T01:42:40,041 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/8a26cef0cd24479ab40c7af7f1c979d3, entries=150, sequenceid=362, filesize=30.5 K 2024-11-28T01:42:40,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/502b0e7568d4424da3b950d37f5b3f48 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/502b0e7568d4424da3b950d37f5b3f48 2024-11-28T01:42:40,046 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/502b0e7568d4424da3b950d37f5b3f48, entries=150, sequenceid=362, filesize=12.0 K 2024-11-28T01:42:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/85bfd0cac5db45a9b0b839bd8aacbb55 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/85bfd0cac5db45a9b0b839bd8aacbb55 2024-11-28T01:42:40,051 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/85bfd0cac5db45a9b0b839bd8aacbb55, entries=150, sequenceid=362, filesize=12.0 K 2024-11-28T01:42:40,052 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9420be97650ba7f85aad07a9b4f2cb6f in 666ms, sequenceid=362, compaction requested=true 2024-11-28T01:42:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-28T01:42:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-28T01:42:40,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-28T01:42:40,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5970 sec 2024-11-28T01:42:40,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.6030 sec 2024-11-28T01:42:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:40,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:42:40,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:40,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:40,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:40,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:40,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:40,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:40,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112811ab2f3c7ee0412a862a700cd78d67b2_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758159481/Put/seqid=0 2024-11-28T01:42:40,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742321_1497 (size=14994) 2024-11-28T01:42:40,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:40,244 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112811ab2f3c7ee0412a862a700cd78d67b2_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112811ab2f3c7ee0412a862a700cd78d67b2_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:40,245 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/2415dd1e9ed84e0e8225d6216f0be754, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:40,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/2415dd1e9ed84e0e8225d6216f0be754 is 175, key is test_row_0/A:col10/1732758159481/Put/seqid=0 2024-11-28T01:42:40,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742322_1498 (size=39949) 2024-11-28T01:42:40,308 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/2415dd1e9ed84e0e8225d6216f0be754 2024-11-28T01:42:40,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7f9801deab7f4f6597477881fdceaf9c is 50, key is test_row_0/B:col10/1732758159481/Put/seqid=0 2024-11-28T01:42:40,377 INFO [Thread-1757 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1757. 2024-11-28T01:42:40,377 DEBUG [Thread-1757 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:40,377 DEBUG [Thread-1757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,377 INFO [Thread-1761 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1761. 2024-11-28T01:42:40,377 DEBUG [Thread-1757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,377 DEBUG [Thread-1761 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,377 DEBUG [Thread-1761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52750 deadline: 1732758220333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:40,377 DEBUG [Thread-1761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:42:40,381 INFO [Thread-1755 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1755. 2024-11-28T01:42:40,381 DEBUG [Thread-1755 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,381 DEBUG [Thread-1755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,381 DEBUG [Thread-1755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742323_1499 (size=12301) 2024-11-28T01:42:40,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7f9801deab7f4f6597477881fdceaf9c 2024-11-28T01:42:40,409 INFO [Thread-1764 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1764. 2024-11-28T01:42:40,409 DEBUG [Thread-1764 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,409 DEBUG [Thread-1764 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,410 DEBUG [Thread-1764 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,412 INFO [Thread-1766 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1766. 2024-11-28T01:42:40,412 DEBUG [Thread-1766 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,412 DEBUG [Thread-1766 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,412 DEBUG [Thread-1766 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,415 INFO [Thread-1772 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1772. 2024-11-28T01:42:40,415 DEBUG [Thread-1772 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,415 DEBUG [Thread-1772 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,415 DEBUG [Thread-1772 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,431 INFO [Thread-1768 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1768. 2024-11-28T01:42:40,431 DEBUG [Thread-1768 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,431 DEBUG [Thread-1768 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,431 DEBUG [Thread-1768 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,436 INFO [Thread-1770 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1770. 2024-11-28T01:42:40,436 DEBUG [Thread-1770 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,436 DEBUG [Thread-1770 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,436 DEBUG [Thread-1770 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a668b58abe48472cb24d22afb6454f56 is 50, key is test_row_0/C:col10/1732758159481/Put/seqid=0 2024-11-28T01:42:40,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742324_1500 (size=12301) 2024-11-28T01:42:40,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a668b58abe48472cb24d22afb6454f56 2024-11-28T01:42:40,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/2415dd1e9ed84e0e8225d6216f0be754 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2415dd1e9ed84e0e8225d6216f0be754 2024-11-28T01:42:40,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2415dd1e9ed84e0e8225d6216f0be754, entries=200, sequenceid=375, filesize=39.0 K 2024-11-28T01:42:40,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/7f9801deab7f4f6597477881fdceaf9c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7f9801deab7f4f6597477881fdceaf9c 2024-11-28T01:42:40,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7f9801deab7f4f6597477881fdceaf9c, entries=150, sequenceid=375, filesize=12.0 K 2024-11-28T01:42:40,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a668b58abe48472cb24d22afb6454f56 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a668b58abe48472cb24d22afb6454f56 2024-11-28T01:42:40,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a668b58abe48472cb24d22afb6454f56, entries=150, sequenceid=375, filesize=12.0 K 2024-11-28T01:42:40,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9420be97650ba7f85aad07a9b4f2cb6f in 307ms, sequenceid=375, compaction requested=true 2024-11-28T01:42:40,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:40,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:40,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:40,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:40,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:40,478 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:40,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9420be97650ba7f85aad07a9b4f2cb6f:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:40,478 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:40,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:40,479 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:40,479 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/B is initiating minor compaction (all files) 2024-11-28T01:42:40,479 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/B in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:40,479 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:40,479 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e4478bd170a64e8d97d9aafa8494347c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7236400131fc4157ad3f01164df7e7a9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/502b0e7568d4424da3b950d37f5b3f48, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7f9801deab7f4f6597477881fdceaf9c] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=48.8 K 2024-11-28T01:42:40,479 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/A is initiating minor compaction (all files) 2024-11-28T01:42:40,479 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/A in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:40,479 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2d827795fc5346d6964c642349c57d5d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/de80fd2dafc64a958a10ab07a253211b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/8a26cef0cd24479ab40c7af7f1c979d3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2415dd1e9ed84e0e8225d6216f0be754] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=139.8 K 2024-11-28T01:42:40,479 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:40,480 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2d827795fc5346d6964c642349c57d5d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/de80fd2dafc64a958a10ab07a253211b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/8a26cef0cd24479ab40c7af7f1c979d3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2415dd1e9ed84e0e8225d6216f0be754] 2024-11-28T01:42:40,480 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e4478bd170a64e8d97d9aafa8494347c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732758155248 2024-11-28T01:42:40,480 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d827795fc5346d6964c642349c57d5d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732758155248 2024-11-28T01:42:40,481 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7236400131fc4157ad3f01164df7e7a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732758157490 2024-11-28T01:42:40,481 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting de80fd2dafc64a958a10ab07a253211b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732758157489 2024-11-28T01:42:40,483 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 502b0e7568d4424da3b950d37f5b3f48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732758158222 2024-11-28T01:42:40,483 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a26cef0cd24479ab40c7af7f1c979d3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732758158222 2024-11-28T01:42:40,484 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2415dd1e9ed84e0e8225d6216f0be754, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732758159481 2024-11-28T01:42:40,484 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f9801deab7f4f6597477881fdceaf9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732758159481 2024-11-28T01:42:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:40,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:42:40,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:40,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:40,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:40,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:40,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:40,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:40,486 INFO [Thread-1753 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1753. 2024-11-28T01:42:40,486 DEBUG [Thread-1753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:40,486 DEBUG [Thread-1753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,486 DEBUG [Thread-1753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:40,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:40,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112824fb6d0246a84ba3ae8bd8c7fb2fad06_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_0/A:col10/1732758160483/Put/seqid=0 2024-11-28T01:42:40,508 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:40,512 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:40,513 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/0dcc579ade2f490a934e62d5905897ad is 50, key is test_row_0/B:col10/1732758159481/Put/seqid=0 2024-11-28T01:42:40,516 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128a2eca74f283a4cf39e58cb29ac770cd1_9420be97650ba7f85aad07a9b4f2cb6f store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:40,532 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128a2eca74f283a4cf39e58cb29ac770cd1_9420be97650ba7f85aad07a9b4f2cb6f, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:40,532 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a2eca74f283a4cf39e58cb29ac770cd1_9420be97650ba7f85aad07a9b4f2cb6f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:40,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742325_1501 (size=12454) 2024-11-28T01:42:40,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742326_1502 (size=13221) 2024-11-28T01:42:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T01:42:40,603 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:40,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742327_1503 (size=4469) 2024-11-28T01:42:40,617 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#A#compaction#419 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:40,617 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/4581b991926a4bd5b2ca3c9d0e213476 is 175, key is test_row_0/A:col10/1732758159481/Put/seqid=0 2024-11-28T01:42:40,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742328_1504 (size=32175) 2024-11-28T01:42:40,950 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:40,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112824fb6d0246a84ba3ae8bd8c7fb2fad06_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112824fb6d0246a84ba3ae8bd8c7fb2fad06_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:40,955 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/42173b100fa0486e9af019a55b813cca, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:40,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/42173b100fa0486e9af019a55b813cca is 175, key is test_row_0/A:col10/1732758160483/Put/seqid=0 2024-11-28T01:42:40,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742329_1505 (size=31255) 2024-11-28T01:42:40,974 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/0dcc579ade2f490a934e62d5905897ad as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/0dcc579ade2f490a934e62d5905897ad 2024-11-28T01:42:40,978 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/B of 9420be97650ba7f85aad07a9b4f2cb6f into 0dcc579ade2f490a934e62d5905897ad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:40,978 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:40,978 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/B, priority=12, startTime=1732758160478; duration=0sec 2024-11-28T01:42:40,978 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:40,978 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:B 2024-11-28T01:42:40,978 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:40,980 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:40,980 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9420be97650ba7f85aad07a9b4f2cb6f/C is initiating minor compaction (all files) 2024-11-28T01:42:40,980 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9420be97650ba7f85aad07a9b4f2cb6f/C in TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:40,980 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e80719074e854516aaac916e558ada4c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e102fa0fc63840d2a3347fe66ff70fa4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/85bfd0cac5db45a9b0b839bd8aacbb55, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a668b58abe48472cb24d22afb6454f56] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp, totalSize=48.8 K 2024-11-28T01:42:40,980 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e80719074e854516aaac916e558ada4c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732758155248 2024-11-28T01:42:40,981 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting e102fa0fc63840d2a3347fe66ff70fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732758157490 2024-11-28T01:42:40,981 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bfd0cac5db45a9b0b839bd8aacbb55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732758158222 2024-11-28T01:42:40,981 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a668b58abe48472cb24d22afb6454f56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732758159481 2024-11-28T01:42:40,996 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9420be97650ba7f85aad07a9b4f2cb6f#C#compaction#421 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:40,997 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/82e609e2277a4144900f3879fb3e99d4 is 50, key is test_row_0/C:col10/1732758159481/Put/seqid=0 2024-11-28T01:42:41,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742330_1506 (size=13221) 2024-11-28T01:42:41,011 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/82e609e2277a4144900f3879fb3e99d4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/82e609e2277a4144900f3879fb3e99d4 2024-11-28T01:42:41,017 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/C of 9420be97650ba7f85aad07a9b4f2cb6f into 82e609e2277a4144900f3879fb3e99d4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:41,017 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:41,017 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/C, priority=12, startTime=1732758160478; duration=0sec 2024-11-28T01:42:41,017 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:41,017 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:C 2024-11-28T01:42:41,027 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/4581b991926a4bd5b2ca3c9d0e213476 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/4581b991926a4bd5b2ca3c9d0e213476 2024-11-28T01:42:41,031 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9420be97650ba7f85aad07a9b4f2cb6f/A of 9420be97650ba7f85aad07a9b4f2cb6f into 4581b991926a4bd5b2ca3c9d0e213476(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:41,031 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:41,031 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., storeName=9420be97650ba7f85aad07a9b4f2cb6f/A, priority=12, startTime=1732758160477; duration=0sec 2024-11-28T01:42:41,032 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:41,032 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9420be97650ba7f85aad07a9b4f2cb6f:A 2024-11-28T01:42:41,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/42173b100fa0486e9af019a55b813cca 2024-11-28T01:42:41,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/76da8c75a2e24d5aa29b93debc530913 is 50, key is test_row_0/B:col10/1732758160483/Put/seqid=0 2024-11-28T01:42:41,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742331_1507 (size=12301) 2024-11-28T01:42:41,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/76da8c75a2e24d5aa29b93debc530913 2024-11-28T01:42:41,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/50d35c238f6146079711f0f944778859 is 50, key is test_row_0/C:col10/1732758160483/Put/seqid=0 2024-11-28T01:42:41,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742332_1508 (size=12301) 2024-11-28T01:42:42,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/50d35c238f6146079711f0f944778859 2024-11-28T01:42:42,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/42173b100fa0486e9af019a55b813cca as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/42173b100fa0486e9af019a55b813cca 2024-11-28T01:42:42,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/42173b100fa0486e9af019a55b813cca, entries=150, sequenceid=395, filesize=30.5 K 2024-11-28T01:42:42,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/76da8c75a2e24d5aa29b93debc530913 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/76da8c75a2e24d5aa29b93debc530913 2024-11-28T01:42:42,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/76da8c75a2e24d5aa29b93debc530913, entries=150, sequenceid=395, filesize=12.0 K 2024-11-28T01:42:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/50d35c238f6146079711f0f944778859 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/50d35c238f6146079711f0f944778859 2024-11-28T01:42:42,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/50d35c238f6146079711f0f944778859, entries=150, sequenceid=395, filesize=12.0 K 2024-11-28T01:42:42,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 9420be97650ba7f85aad07a9b4f2cb6f in 1712ms, sequenceid=395, compaction requested=false 2024-11-28T01:42:42,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9420be97650ba7f85aad07a9b4f2cb6f: 2024-11-28T01:42:42,544 INFO [Thread-1759 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1759. 2024-11-28T01:42:42,544 DEBUG [Thread-1759 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:42:42,544 DEBUG [Thread-1759 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:42,544 DEBUG [Thread-1759 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:42,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:42,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2937 rows 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 983 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2933 rows 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 985 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2946 rows 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2934 rows 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 988 2024-11-28T01:42:42,545 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2954 rows 2024-11-28T01:42:42,545 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:42:42,545 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:42:42,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:42,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:42,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:42:42,546 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T01:42:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T01:42:42,549 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758162549"}]},"ts":"1732758162549"} 2024-11-28T01:42:42,550 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T01:42:42,550 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T01:42:42,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:42:42,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, UNASSIGN}] 2024-11-28T01:42:42,552 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, UNASSIGN 2024-11-28T01:42:42,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:42,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, UNASSIGN because future has completed 2024-11-28T01:42:42,554 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:42:42,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:42:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T01:42:42,706 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:42,706 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:42:42,706 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 9420be97650ba7f85aad07a9b4f2cb6f, disabling compactions & flushes 2024-11-28T01:42:42,706 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:42,706 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:42,706 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. after waiting 0 ms 2024-11-28T01:42:42,706 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:42,707 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 9420be97650ba7f85aad07a9b4f2cb6f 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-28T01:42:42,707 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=A 2024-11-28T01:42:42,707 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:42,707 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=B 2024-11-28T01:42:42,707 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:42,707 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9420be97650ba7f85aad07a9b4f2cb6f, store=C 2024-11-28T01:42:42,707 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:42,711 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d4099fa105ba4bd282428ed61ec003bb_9420be97650ba7f85aad07a9b4f2cb6f is 50, key is test_row_2/A:col10/1732758162543/Put/seqid=0 2024-11-28T01:42:42,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742333_1509 (size=7374) 2024-11-28T01:42:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T01:42:43,115 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:43,118 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d4099fa105ba4bd282428ed61ec003bb_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4099fa105ba4bd282428ed61ec003bb_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:43,119 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b14ff906749d44ef826841a09b055fd9, store: [table=TestAcidGuarantees family=A region=9420be97650ba7f85aad07a9b4f2cb6f] 2024-11-28T01:42:43,119 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b14ff906749d44ef826841a09b055fd9 is 175, key is test_row_2/A:col10/1732758162543/Put/seqid=0 2024-11-28T01:42:43,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742334_1510 (size=13865) 2024-11-28T01:42:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T01:42:43,523 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b14ff906749d44ef826841a09b055fd9 2024-11-28T01:42:43,528 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/8aa2a511596f4df88b603fda53229ad6 is 50, key is test_row_2/B:col10/1732758162543/Put/seqid=0 2024-11-28T01:42:43,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742335_1511 (size=7415) 2024-11-28T01:42:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T01:42:43,932 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/8aa2a511596f4df88b603fda53229ad6 2024-11-28T01:42:43,937 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a5e1172f85ab42008705484b556507a0 is 50, key is test_row_2/C:col10/1732758162543/Put/seqid=0 2024-11-28T01:42:43,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742336_1512 (size=7415) 2024-11-28T01:42:44,342 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a5e1172f85ab42008705484b556507a0 2024-11-28T01:42:44,345 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/A/b14ff906749d44ef826841a09b055fd9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b14ff906749d44ef826841a09b055fd9 2024-11-28T01:42:44,348 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b14ff906749d44ef826841a09b055fd9, entries=50, sequenceid=402, filesize=13.5 K 2024-11-28T01:42:44,349 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/B/8aa2a511596f4df88b603fda53229ad6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/8aa2a511596f4df88b603fda53229ad6 2024-11-28T01:42:44,352 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/8aa2a511596f4df88b603fda53229ad6, entries=50, sequenceid=402, filesize=7.2 K 2024-11-28T01:42:44,352 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/.tmp/C/a5e1172f85ab42008705484b556507a0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a5e1172f85ab42008705484b556507a0 2024-11-28T01:42:44,355 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a5e1172f85ab42008705484b556507a0, entries=50, sequenceid=402, filesize=7.2 K 2024-11-28T01:42:44,355 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 9420be97650ba7f85aad07a9b4f2cb6f in 1649ms, sequenceid=402, compaction requested=true 2024-11-28T01:42:44,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/05b3177f60f84ee4ab65aabd564c8135, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/78da5ee50d6e4039a670671c4c269e86, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1c098bcbf1e146c1983b4355931fafe5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bfded09129e74d26aecc6ec312a845e7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/c4dc6c8e3d25497f96a80635d76b9778, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d192eabe277147d8bf455458e0aa87ab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e6603afae6a4a2b9016a90723071695, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/be1c0c212fe54e219c217865799f13ae, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/fe320457286e4787a7dc162a382f6bf3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/87b339f8ffae49a189dd78fd8f223965, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/9f69babe0c4b4201b6de60805de73d02, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d769985b7ef4478a8a40ef691e14a9b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/eb870154d2dc4180b2598108c2f8a55b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bdc061c23cca4483a0dd6e02fbeba723, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/036fb36fd17947119cec983cc2bb5ec4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b32612da26ec4dd2a92af8d914760bcc, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/261087f33d2c4df09905da37f4eb7373, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/a8c30ea47e814e10b41da32beeea67c5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/f37979cdfed949fc94e4a8cd34772e9f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d9ae81aee3ef4095b7da6600dd37c66b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e732e08bca74daf9cb442bbc758bbcd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1f6327150a0b437d86084bf49046c6b2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/5ebaa00b30474332acc56913e361b801, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2d827795fc5346d6964c642349c57d5d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/de80fd2dafc64a958a10ab07a253211b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/8a26cef0cd24479ab40c7af7f1c979d3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2415dd1e9ed84e0e8225d6216f0be754] to archive 2024-11-28T01:42:44,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:44,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/05b3177f60f84ee4ab65aabd564c8135 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/05b3177f60f84ee4ab65aabd564c8135 2024-11-28T01:42:44,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/78da5ee50d6e4039a670671c4c269e86 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/78da5ee50d6e4039a670671c4c269e86 2024-11-28T01:42:44,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1c098bcbf1e146c1983b4355931fafe5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1c098bcbf1e146c1983b4355931fafe5 2024-11-28T01:42:44,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bfded09129e74d26aecc6ec312a845e7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bfded09129e74d26aecc6ec312a845e7 2024-11-28T01:42:44,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/c4dc6c8e3d25497f96a80635d76b9778 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/c4dc6c8e3d25497f96a80635d76b9778 2024-11-28T01:42:44,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d192eabe277147d8bf455458e0aa87ab to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d192eabe277147d8bf455458e0aa87ab 2024-11-28T01:42:44,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e6603afae6a4a2b9016a90723071695 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e6603afae6a4a2b9016a90723071695 2024-11-28T01:42:44,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/be1c0c212fe54e219c217865799f13ae to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/be1c0c212fe54e219c217865799f13ae 2024-11-28T01:42:44,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/fe320457286e4787a7dc162a382f6bf3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/fe320457286e4787a7dc162a382f6bf3 2024-11-28T01:42:44,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/87b339f8ffae49a189dd78fd8f223965 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/87b339f8ffae49a189dd78fd8f223965 2024-11-28T01:42:44,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/9f69babe0c4b4201b6de60805de73d02 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/9f69babe0c4b4201b6de60805de73d02 2024-11-28T01:42:44,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d769985b7ef4478a8a40ef691e14a9b1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d769985b7ef4478a8a40ef691e14a9b1 2024-11-28T01:42:44,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/eb870154d2dc4180b2598108c2f8a55b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/eb870154d2dc4180b2598108c2f8a55b 2024-11-28T01:42:44,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bdc061c23cca4483a0dd6e02fbeba723 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/bdc061c23cca4483a0dd6e02fbeba723 2024-11-28T01:42:44,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/036fb36fd17947119cec983cc2bb5ec4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/036fb36fd17947119cec983cc2bb5ec4 2024-11-28T01:42:44,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b32612da26ec4dd2a92af8d914760bcc to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b32612da26ec4dd2a92af8d914760bcc 2024-11-28T01:42:44,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/261087f33d2c4df09905da37f4eb7373 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/261087f33d2c4df09905da37f4eb7373 2024-11-28T01:42:44,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/a8c30ea47e814e10b41da32beeea67c5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/a8c30ea47e814e10b41da32beeea67c5 2024-11-28T01:42:44,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/f37979cdfed949fc94e4a8cd34772e9f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/f37979cdfed949fc94e4a8cd34772e9f 2024-11-28T01:42:44,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d9ae81aee3ef4095b7da6600dd37c66b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/d9ae81aee3ef4095b7da6600dd37c66b 2024-11-28T01:42:44,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e732e08bca74daf9cb442bbc758bbcd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/6e732e08bca74daf9cb442bbc758bbcd 2024-11-28T01:42:44,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1f6327150a0b437d86084bf49046c6b2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/1f6327150a0b437d86084bf49046c6b2 2024-11-28T01:42:44,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/5ebaa00b30474332acc56913e361b801 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/5ebaa00b30474332acc56913e361b801 2024-11-28T01:42:44,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2d827795fc5346d6964c642349c57d5d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2d827795fc5346d6964c642349c57d5d 2024-11-28T01:42:44,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/de80fd2dafc64a958a10ab07a253211b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/de80fd2dafc64a958a10ab07a253211b 2024-11-28T01:42:44,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/8a26cef0cd24479ab40c7af7f1c979d3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/8a26cef0cd24479ab40c7af7f1c979d3 2024-11-28T01:42:44,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2415dd1e9ed84e0e8225d6216f0be754 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/2415dd1e9ed84e0e8225d6216f0be754 2024-11-28T01:42:44,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/ca2c2da397b740b1bd73570ce483b527, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/aa9e2069f9cc4b36aa18f3a6774e61e6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c274595760ac44eeb28d1b1bbd714960, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/6eb7722538594ab9a099421fed8c5235, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/def9cca553724a9594897d5a00897542, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/74fff2bbaae14beb9bec5d9cb15242eb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/54142080464f4252b1cca63648025202, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/cc80ec0eb91b4a5b9f38a31ac7939ab3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2d9d98a6fd52421198492c00005472d6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e627c4892bff442593da108c53247898, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7e5702831b3d4feea9afac6635d3df91, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/f742901a4d3d4e969f45195cbacb98e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a3eb9c7331f74b9daa36468dd55cd2c1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7de67a2a3e5c45649f12ae60ba32b6f1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c83e8b6682cd4566a8b412d3973dbc2a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a47a69167aec4131bfcddaf6ce4163e8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/752edd99cc69460b928c506eb241ec90, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/dc7b5980d5584d8a9d9f0bc2ce1806e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/578eeb0b2fcf4f8d8fac747609cea4cd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/4b80927090ea41dca8f687c1bfa301ff, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2bff0f4b9dc34f988679e133162fab1f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/30869f82eeb1431c87484cae817f2a24, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e4478bd170a64e8d97d9aafa8494347c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/50de29f606bc49ce95704c67f208db78, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7236400131fc4157ad3f01164df7e7a9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/502b0e7568d4424da3b950d37f5b3f48, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7f9801deab7f4f6597477881fdceaf9c] to archive 2024-11-28T01:42:44,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:44,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/ca2c2da397b740b1bd73570ce483b527 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/ca2c2da397b740b1bd73570ce483b527 2024-11-28T01:42:44,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/aa9e2069f9cc4b36aa18f3a6774e61e6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/aa9e2069f9cc4b36aa18f3a6774e61e6 2024-11-28T01:42:44,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c274595760ac44eeb28d1b1bbd714960 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c274595760ac44eeb28d1b1bbd714960 2024-11-28T01:42:44,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/6eb7722538594ab9a099421fed8c5235 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/6eb7722538594ab9a099421fed8c5235 2024-11-28T01:42:44,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/def9cca553724a9594897d5a00897542 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/def9cca553724a9594897d5a00897542 2024-11-28T01:42:44,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/74fff2bbaae14beb9bec5d9cb15242eb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/74fff2bbaae14beb9bec5d9cb15242eb 2024-11-28T01:42:44,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/54142080464f4252b1cca63648025202 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/54142080464f4252b1cca63648025202 2024-11-28T01:42:44,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/cc80ec0eb91b4a5b9f38a31ac7939ab3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/cc80ec0eb91b4a5b9f38a31ac7939ab3 2024-11-28T01:42:44,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2d9d98a6fd52421198492c00005472d6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2d9d98a6fd52421198492c00005472d6 2024-11-28T01:42:44,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e627c4892bff442593da108c53247898 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e627c4892bff442593da108c53247898 2024-11-28T01:42:44,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7e5702831b3d4feea9afac6635d3df91 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7e5702831b3d4feea9afac6635d3df91 2024-11-28T01:42:44,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/f742901a4d3d4e969f45195cbacb98e4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/f742901a4d3d4e969f45195cbacb98e4 2024-11-28T01:42:44,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a3eb9c7331f74b9daa36468dd55cd2c1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a3eb9c7331f74b9daa36468dd55cd2c1 2024-11-28T01:42:44,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7de67a2a3e5c45649f12ae60ba32b6f1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7de67a2a3e5c45649f12ae60ba32b6f1 2024-11-28T01:42:44,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c83e8b6682cd4566a8b412d3973dbc2a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/c83e8b6682cd4566a8b412d3973dbc2a 2024-11-28T01:42:44,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a47a69167aec4131bfcddaf6ce4163e8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/a47a69167aec4131bfcddaf6ce4163e8 2024-11-28T01:42:44,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/752edd99cc69460b928c506eb241ec90 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/752edd99cc69460b928c506eb241ec90 2024-11-28T01:42:44,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/dc7b5980d5584d8a9d9f0bc2ce1806e3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/dc7b5980d5584d8a9d9f0bc2ce1806e3 2024-11-28T01:42:44,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/578eeb0b2fcf4f8d8fac747609cea4cd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/578eeb0b2fcf4f8d8fac747609cea4cd 2024-11-28T01:42:44,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/4b80927090ea41dca8f687c1bfa301ff to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/4b80927090ea41dca8f687c1bfa301ff 2024-11-28T01:42:44,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2bff0f4b9dc34f988679e133162fab1f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/2bff0f4b9dc34f988679e133162fab1f 2024-11-28T01:42:44,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/30869f82eeb1431c87484cae817f2a24 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/30869f82eeb1431c87484cae817f2a24 2024-11-28T01:42:44,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e4478bd170a64e8d97d9aafa8494347c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/e4478bd170a64e8d97d9aafa8494347c 2024-11-28T01:42:44,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/50de29f606bc49ce95704c67f208db78 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/50de29f606bc49ce95704c67f208db78 2024-11-28T01:42:44,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7236400131fc4157ad3f01164df7e7a9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7236400131fc4157ad3f01164df7e7a9 2024-11-28T01:42:44,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/502b0e7568d4424da3b950d37f5b3f48 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/502b0e7568d4424da3b950d37f5b3f48 2024-11-28T01:42:44,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7f9801deab7f4f6597477881fdceaf9c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/7f9801deab7f4f6597477881fdceaf9c 2024-11-28T01:42:44,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/94d3c483be764e36b2e781c6d829eb93, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3b2b3d7bd0ab48579284f97814fa40b0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dc90df4c06f642fabc395ad3043d13ca, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/0088d599e4834a93a40045f4c6ddcc7e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9f3e95f06f2946cab14e4d3ed24daeb8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/8578c06b05734b259c0d3c313acbf2d7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e6ca2ccbd1504b31aff79c643e3f9ad9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4ab14fca3906406fb278363631ba4fcf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dde7ba53bc8f4769ab38d12fbdb6f515, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/12a13be6e97f4d238850a8fa5f2e0884, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/abc3c47e34c14d058df890d933ae6d51, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/d28b6da5abc441c1985d75e9b3368276, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/298dc8be054048c0b880e4069f0582bf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/384c4bd1464a450fa3d1000f9511de11, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/ea6494a62b024afdba117e13dbe2bf27, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/619ec4647e1b49c5984b15cf8ea2a31c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/14bf2f695621495183e50d957ad9d3a6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9adbcefcd4784950b90c2a5fb9741e22, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3caee540f7184737a4029ae4c67bd9d2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/08e1ddc38a4b4d4bb659483be99fd47e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/1bc73086d06342fa8dc5f574c8887099, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4b79161cc3e443d0a3d6771641029462, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e80719074e854516aaac916e558ada4c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a9851025c4b347c7baf73f31b7d8f6ac, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e102fa0fc63840d2a3347fe66ff70fa4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/85bfd0cac5db45a9b0b839bd8aacbb55, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a668b58abe48472cb24d22afb6454f56] to archive 2024-11-28T01:42:44,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:44,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/94d3c483be764e36b2e781c6d829eb93 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/94d3c483be764e36b2e781c6d829eb93 2024-11-28T01:42:44,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3b2b3d7bd0ab48579284f97814fa40b0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3b2b3d7bd0ab48579284f97814fa40b0 2024-11-28T01:42:44,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dc90df4c06f642fabc395ad3043d13ca to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dc90df4c06f642fabc395ad3043d13ca 2024-11-28T01:42:44,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/0088d599e4834a93a40045f4c6ddcc7e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/0088d599e4834a93a40045f4c6ddcc7e 2024-11-28T01:42:44,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9f3e95f06f2946cab14e4d3ed24daeb8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9f3e95f06f2946cab14e4d3ed24daeb8 2024-11-28T01:42:44,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/8578c06b05734b259c0d3c313acbf2d7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/8578c06b05734b259c0d3c313acbf2d7 2024-11-28T01:42:44,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e6ca2ccbd1504b31aff79c643e3f9ad9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e6ca2ccbd1504b31aff79c643e3f9ad9 2024-11-28T01:42:44,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4ab14fca3906406fb278363631ba4fcf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4ab14fca3906406fb278363631ba4fcf 2024-11-28T01:42:44,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dde7ba53bc8f4769ab38d12fbdb6f515 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/dde7ba53bc8f4769ab38d12fbdb6f515 2024-11-28T01:42:44,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/12a13be6e97f4d238850a8fa5f2e0884 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/12a13be6e97f4d238850a8fa5f2e0884 2024-11-28T01:42:44,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/abc3c47e34c14d058df890d933ae6d51 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/abc3c47e34c14d058df890d933ae6d51 2024-11-28T01:42:44,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/d28b6da5abc441c1985d75e9b3368276 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/d28b6da5abc441c1985d75e9b3368276 2024-11-28T01:42:44,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/298dc8be054048c0b880e4069f0582bf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/298dc8be054048c0b880e4069f0582bf 2024-11-28T01:42:44,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/384c4bd1464a450fa3d1000f9511de11 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/384c4bd1464a450fa3d1000f9511de11 2024-11-28T01:42:44,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/ea6494a62b024afdba117e13dbe2bf27 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/ea6494a62b024afdba117e13dbe2bf27 2024-11-28T01:42:44,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/619ec4647e1b49c5984b15cf8ea2a31c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/619ec4647e1b49c5984b15cf8ea2a31c 2024-11-28T01:42:44,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/14bf2f695621495183e50d957ad9d3a6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/14bf2f695621495183e50d957ad9d3a6 2024-11-28T01:42:44,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9adbcefcd4784950b90c2a5fb9741e22 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/9adbcefcd4784950b90c2a5fb9741e22 2024-11-28T01:42:44,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3caee540f7184737a4029ae4c67bd9d2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/3caee540f7184737a4029ae4c67bd9d2 2024-11-28T01:42:44,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/08e1ddc38a4b4d4bb659483be99fd47e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/08e1ddc38a4b4d4bb659483be99fd47e 2024-11-28T01:42:44,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/1bc73086d06342fa8dc5f574c8887099 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/1bc73086d06342fa8dc5f574c8887099 2024-11-28T01:42:44,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4b79161cc3e443d0a3d6771641029462 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/4b79161cc3e443d0a3d6771641029462 2024-11-28T01:42:44,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e80719074e854516aaac916e558ada4c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e80719074e854516aaac916e558ada4c 2024-11-28T01:42:44,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a9851025c4b347c7baf73f31b7d8f6ac to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a9851025c4b347c7baf73f31b7d8f6ac 2024-11-28T01:42:44,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e102fa0fc63840d2a3347fe66ff70fa4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/e102fa0fc63840d2a3347fe66ff70fa4 2024-11-28T01:42:44,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/85bfd0cac5db45a9b0b839bd8aacbb55 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/85bfd0cac5db45a9b0b839bd8aacbb55 2024-11-28T01:42:44,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a668b58abe48472cb24d22afb6454f56 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a668b58abe48472cb24d22afb6454f56 2024-11-28T01:42:44,433 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/recovered.edits/405.seqid, newMaxSeqId=405, maxSeqId=4 2024-11-28T01:42:44,434 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. 2024-11-28T01:42:44,434 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 9420be97650ba7f85aad07a9b4f2cb6f: Waiting for close lock at 1732758162706Running coprocessor pre-close hooks at 1732758162706Disabling compacts and flushes for region at 1732758162706Disabling writes for close at 1732758162706Obtaining lock to block concurrent updates at 1732758162707 (+1 ms)Preparing flush snapshotting stores in 9420be97650ba7f85aad07a9b4f2cb6f at 1732758162707Finished memstore snapshotting TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732758162707Flushing stores of TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f. at 1732758162707Flushing 9420be97650ba7f85aad07a9b4f2cb6f/A: creating writer at 1732758162707Flushing mob file 9420be97650ba7f85aad07a9b4f2cb6f/A: appending metadata at 1732758162711 (+4 ms)Flushing mob file 9420be97650ba7f85aad07a9b4f2cb6f/A: closing flushed file at 1732758162711Flushing 9420be97650ba7f85aad07a9b4f2cb6f/A: appending metadata at 1732758163119 (+408 ms)Flushing 9420be97650ba7f85aad07a9b4f2cb6f/A: closing flushed file at 1732758163119Flushing 9420be97650ba7f85aad07a9b4f2cb6f/B: creating writer at 1732758163526 (+407 ms)Flushing 9420be97650ba7f85aad07a9b4f2cb6f/B: appending metadata at 1732758163528 (+2 ms)Flushing 9420be97650ba7f85aad07a9b4f2cb6f/B: closing flushed file at 1732758163528Flushing 9420be97650ba7f85aad07a9b4f2cb6f/C: creating writer at 1732758163935 (+407 ms)Flushing 9420be97650ba7f85aad07a9b4f2cb6f/C: appending metadata at 1732758163937 (+2 ms)Flushing 9420be97650ba7f85aad07a9b4f2cb6f/C: closing flushed file at 1732758163937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@646fe0ce: reopening flushed file at 1732758164345 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a3dac8d: reopening flushed file at 1732758164348 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@351ce609: reopening flushed file at 1732758164352 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 9420be97650ba7f85aad07a9b4f2cb6f in 1649ms, sequenceid=402, compaction requested=true at 1732758164355 (+3 ms)Writing region close event to WAL at 1732758164430 (+75 ms)Running coprocessor post-close hooks at 1732758164434 (+4 ms)Closed at 1732758164434 2024-11-28T01:42:44,436 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,436 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=9420be97650ba7f85aad07a9b4f2cb6f, regionState=CLOSED 2024-11-28T01:42:44,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:42:44,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-28T01:42:44,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 9420be97650ba7f85aad07a9b4f2cb6f, server=9df3f25f5867,42371,1732758050017 in 1.8840 sec 2024-11-28T01:42:44,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-28T01:42:44,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9420be97650ba7f85aad07a9b4f2cb6f, UNASSIGN in 1.8880 sec 2024-11-28T01:42:44,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-28T01:42:44,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8900 sec 2024-11-28T01:42:44,444 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758164443"}]},"ts":"1732758164443"} 2024-11-28T01:42:44,445 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T01:42:44,445 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T01:42:44,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8990 sec 2024-11-28T01:42:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T01:42:44,692 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:44,693 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T01:42:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:44,695 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T01:42:44,695 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:44,698 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,700 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/recovered.edits] 2024-11-28T01:42:44,702 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/42173b100fa0486e9af019a55b813cca to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/42173b100fa0486e9af019a55b813cca 2024-11-28T01:42:44,703 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/4581b991926a4bd5b2ca3c9d0e213476 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/4581b991926a4bd5b2ca3c9d0e213476 2024-11-28T01:42:44,704 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b14ff906749d44ef826841a09b055fd9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/A/b14ff906749d44ef826841a09b055fd9 2024-11-28T01:42:44,705 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/0dcc579ade2f490a934e62d5905897ad to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/0dcc579ade2f490a934e62d5905897ad 2024-11-28T01:42:44,706 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/76da8c75a2e24d5aa29b93debc530913 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/76da8c75a2e24d5aa29b93debc530913 2024-11-28T01:42:44,707 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/8aa2a511596f4df88b603fda53229ad6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/B/8aa2a511596f4df88b603fda53229ad6 2024-11-28T01:42:44,709 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/50d35c238f6146079711f0f944778859 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/50d35c238f6146079711f0f944778859 2024-11-28T01:42:44,710 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/82e609e2277a4144900f3879fb3e99d4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/82e609e2277a4144900f3879fb3e99d4 2024-11-28T01:42:44,711 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a5e1172f85ab42008705484b556507a0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/C/a5e1172f85ab42008705484b556507a0 2024-11-28T01:42:44,713 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/recovered.edits/405.seqid to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f/recovered.edits/405.seqid 2024-11-28T01:42:44,714 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,714 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T01:42:44,714 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T01:42:44,715 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T01:42:44,717 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112806aef69b37f24dd7957231bcc3595333_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112806aef69b37f24dd7957231bcc3595333_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,718 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112811ab2f3c7ee0412a862a700cd78d67b2_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112811ab2f3c7ee0412a862a700cd78d67b2_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,719 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128124e0660c4414b13a7f6744f08f18f2b_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128124e0660c4414b13a7f6744f08f18f2b_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,720 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281c09570b79544c9a9ba891d5b1f21498_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281c09570b79544c9a9ba891d5b1f21498_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,720 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128201a61306f9543ae98716c243424c774_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128201a61306f9543ae98716c243424c774_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,721 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112824fb6d0246a84ba3ae8bd8c7fb2fad06_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112824fb6d0246a84ba3ae8bd8c7fb2fad06_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,722 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f5cb5e8f0f144cdbeab7ca9be242bb5_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f5cb5e8f0f144cdbeab7ca9be242bb5_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,723 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128498192fe83144126ae61a8909177559f_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128498192fe83144126ae61a8909177559f_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,724 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284d9087ece76d4de8831d04ff7f213fab_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284d9087ece76d4de8831d04ff7f213fab_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,725 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411286f8cb0b75c1a4372979bc4408156eb4c_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411286f8cb0b75c1a4372979bc4408156eb4c_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,726 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128728248d0aa7f4179aea0a694efd3267c_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128728248d0aa7f4179aea0a694efd3267c_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,727 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287e878a9c6abb4095b19f7a5c0b50fcd4_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287e878a9c6abb4095b19f7a5c0b50fcd4_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,728 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112885cf8b56a20f4a1a9403d541741093a7_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112885cf8b56a20f4a1a9403d541741093a7_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,729 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288b2eb6d6098448d9a9f0607417cf91e9_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288b2eb6d6098448d9a9f0607417cf91e9_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,730 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b145d5c49fa64e84a88bd6902a505af8_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b145d5c49fa64e84a88bd6902a505af8_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bcb26c05f3d444a4b5c14d70e4e89d54_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bcb26c05f3d444a4b5c14d70e4e89d54_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,732 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4099fa105ba4bd282428ed61ec003bb_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4099fa105ba4bd282428ed61ec003bb_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,733 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d5bc0c401f984102bf7e48b47b24c0db_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d5bc0c401f984102bf7e48b47b24c0db_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,734 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128db9d4c46250b4882aa00eb45bab9c9cf_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128db9d4c46250b4882aa00eb45bab9c9cf_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,735 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e3540a19809d4e00b5098fa0858569c3_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e3540a19809d4e00b5098fa0858569c3_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,735 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f340c6b8a88d467fae120f6d29bead3b_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f340c6b8a88d467fae120f6d29bead3b_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,736 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fdc35afc070d49a1963961d2023420b5_9420be97650ba7f85aad07a9b4f2cb6f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128fdc35afc070d49a1963961d2023420b5_9420be97650ba7f85aad07a9b4f2cb6f 2024-11-28T01:42:44,737 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T01:42:44,738 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:44,740 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T01:42:44,742 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T01:42:44,743 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:44,743 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T01:42:44,743 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732758164743"}]},"ts":"9223372036854775807"} 2024-11-28T01:42:44,745 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T01:42:44,745 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9420be97650ba7f85aad07a9b4f2cb6f, NAME => 'TestAcidGuarantees,,1732758137233.9420be97650ba7f85aad07a9b4f2cb6f.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T01:42:44,745 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T01:42:44,745 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732758164745"}]},"ts":"9223372036854775807"} 2024-11-28T01:42:44,747 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T01:42:44,747 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:44,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-28T01:42:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T01:42:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T01:42:44,803 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:44,813 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_752253615_22 at /127.0.0.1:47898 [Waiting for operation #925] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1056209077_22 at /127.0.0.1:48020 [Waiting for operation #1286] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_752253615_22 at /127.0.0.1:47842 [Waiting for operation #932] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1056209077_22 at /127.0.0.1:47914 [Waiting for operation #908] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=639 (was 641), ProcessCount=11 (was 11), AvailableMemoryMB=6627 (was 7216) 2024-11-28T01:42:44,822 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=639, ProcessCount=11, AvailableMemoryMB=6627 2024-11-28T01:42:44,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:42:44,823 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:42:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T01:42:44,825 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:42:44,825 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:44,825 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-28T01:42:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T01:42:44,826 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:42:44,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742337_1513 (size=1061) 2024-11-28T01:42:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T01:42:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T01:42:45,233 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:42:45,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742338_1514 (size=53) 2024-11-28T01:42:45,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:42:45,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9f37285715eee60df10bafa1eb0d60a0, disabling compactions & flushes 2024-11-28T01:42:45,241 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. after waiting 0 ms 2024-11-28T01:42:45,241 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,241 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,242 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9f37285715eee60df10bafa1eb0d60a0: Waiting for close lock at 1732758165241Disabling compacts and flushes for region at 1732758165241Disabling writes for close at 1732758165241Writing region close event to WAL at 1732758165241Closed at 1732758165241 2024-11-28T01:42:45,242 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:42:45,243 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732758165243"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758165243"}]},"ts":"1732758165243"} 2024-11-28T01:42:45,244 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:42:45,244 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:42:45,245 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758165244"}]},"ts":"1732758165244"} 2024-11-28T01:42:45,245 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T01:42:45,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, ASSIGN}] 2024-11-28T01:42:45,246 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, ASSIGN 2024-11-28T01:42:45,247 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, ASSIGN; state=OFFLINE, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=false 2024-11-28T01:42:45,398 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=9f37285715eee60df10bafa1eb0d60a0, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:45,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, ASSIGN because future has completed 2024-11-28T01:42:45,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:42:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T01:42:45,554 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,554 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:42:45,555 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,555 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:42:45,555 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,555 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,556 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,557 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:45,557 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f37285715eee60df10bafa1eb0d60a0 columnFamilyName A 2024-11-28T01:42:45,557 DEBUG [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:45,558 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.HStore(327): Store=9f37285715eee60df10bafa1eb0d60a0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:45,558 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,559 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:45,559 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f37285715eee60df10bafa1eb0d60a0 columnFamilyName B 2024-11-28T01:42:45,559 DEBUG [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:45,559 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.HStore(327): Store=9f37285715eee60df10bafa1eb0d60a0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:45,559 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,560 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:42:45,560 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f37285715eee60df10bafa1eb0d60a0 columnFamilyName C 2024-11-28T01:42:45,560 DEBUG [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:42:45,561 INFO [StoreOpener-9f37285715eee60df10bafa1eb0d60a0-1 {}] regionserver.HStore(327): Store=9f37285715eee60df10bafa1eb0d60a0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:42:45,561 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,561 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,561 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,562 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,562 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,563 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,563 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:42:45,564 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,565 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:42:45,566 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 9f37285715eee60df10bafa1eb0d60a0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65432898, jitterRate=-0.02497383952140808}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:42:45,566 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:45,566 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 9f37285715eee60df10bafa1eb0d60a0: Running coprocessor pre-open hook at 1732758165555Writing region info on filesystem at 1732758165555Initializing all the Stores at 1732758165556 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758165556Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758165556Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758165556Cleaning up temporary data from old regions at 1732758165563 (+7 ms)Running coprocessor post-open hooks at 1732758165566 (+3 ms)Region opened successfully at 1732758165566 2024-11-28T01:42:45,567 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., pid=120, masterSystemTime=1732758165551 2024-11-28T01:42:45,568 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,568 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:45,569 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=9f37285715eee60df10bafa1eb0d60a0, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:45,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:42:45,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-28T01:42:45,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 in 171 msec 2024-11-28T01:42:45,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-28T01:42:45,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, ASSIGN in 326 msec 2024-11-28T01:42:45,574 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:42:45,574 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758165574"}]},"ts":"1732758165574"} 2024-11-28T01:42:45,575 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T01:42:45,576 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:42:45,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 753 msec 2024-11-28T01:42:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T01:42:45,963 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:45,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@389d3e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,965 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e39fc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,967 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32428a3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,967 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:45,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:45,970 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:42:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,45903,1732758049085 2024-11-28T01:42:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@547bb5e0 2024-11-28T01:42:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:42:45,971 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:42:45,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,973 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b39eb88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,975 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59259953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,976 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa732fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,978 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e92ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,979 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763a008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,981 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c4ef97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,982 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e74dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,983 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4947d2b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,985 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@274e662d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,986 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78777f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,988 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ddb2417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,989 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c357a9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,990 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f95b28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,991 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639dc50a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,993 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ecf645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,994 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14309b2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,996 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c1c450, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,997 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82ca589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:42:45,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:42:45,998 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:42:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:42:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:42:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19cd661e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:42:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:42:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:42:45,999 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:42:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e2483a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:42:46,000 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,000 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,001 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,002 DEBUG [Thread-2274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,002 DEBUG [Thread-2272 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-28T01:42:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T01:42:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,005 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:46,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:46,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,008 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,008 DEBUG [Thread-2281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,010 DEBUG [Thread-2270 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,011 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,012 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:42:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:42:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:42:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:42:46,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2] 2024-11-28T01:42:46,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:42:46,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:46,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:46,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:46,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:46,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:46,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:46,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/bcff672b7e0e41e68417e9b2fd9fb52e is 50, key is test_row_0/A:col10/1732758166018/Put/seqid=0 2024-11-28T01:42:46,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742339_1515 (size=12001) 2024-11-28T01:42:46,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/bcff672b7e0e41e68417e9b2fd9fb52e 2024-11-28T01:42:46,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758226055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758226056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758226063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758226072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758226073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/0db253bec1914cef99c6d3c4158fb62d is 50, key is test_row_0/B:col10/1732758166018/Put/seqid=0 2024-11-28T01:42:46,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742340_1516 (size=12001) 2024-11-28T01:42:46,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/0db253bec1914cef99c6d3c4158fb62d 2024-11-28T01:42:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T01:42:46,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/964cc5b9b4a54e09bcd98dad4211db0b is 50, key is test_row_0/C:col10/1732758166018/Put/seqid=0 2024-11-28T01:42:46,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742341_1517 (size=12001) 2024-11-28T01:42:46,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/964cc5b9b4a54e09bcd98dad4211db0b 2024-11-28T01:42:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/bcff672b7e0e41e68417e9b2fd9fb52e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/bcff672b7e0e41e68417e9b2fd9fb52e 2024-11-28T01:42:46,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/bcff672b7e0e41e68417e9b2fd9fb52e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T01:42:46,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/0db253bec1914cef99c6d3c4158fb62d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/0db253bec1914cef99c6d3c4158fb62d 2024-11-28T01:42:46,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/0db253bec1914cef99c6d3c4158fb62d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T01:42:46,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/964cc5b9b4a54e09bcd98dad4211db0b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/964cc5b9b4a54e09bcd98dad4211db0b 2024-11-28T01:42:46,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/964cc5b9b4a54e09bcd98dad4211db0b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T01:42:46,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9f37285715eee60df10bafa1eb0d60a0 in 133ms, sequenceid=15, compaction requested=false 2024-11-28T01:42:46,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:46,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T01:42:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:46,162 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T01:42:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:46,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/da0392a3cd2947b5add6885e62fdc45e is 50, key is test_row_0/A:col10/1732758166049/Put/seqid=0 2024-11-28T01:42:46,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742342_1518 (size=12001) 2024-11-28T01:42:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:46,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:46,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758226205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758226210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758226211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758226211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758226219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T01:42:46,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758226323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758226333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758226333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758226333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758226333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758226542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758226552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758226552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758226552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758226553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,589 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/da0392a3cd2947b5add6885e62fdc45e 2024-11-28T01:42:46,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/249a9ac6aeaa4f69a347a25142ba6ded is 50, key is test_row_0/B:col10/1732758166049/Put/seqid=0 2024-11-28T01:42:46,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742343_1519 (size=12001) 2024-11-28T01:42:46,601 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/249a9ac6aeaa4f69a347a25142ba6ded 2024-11-28T01:42:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9464c6d3a2a447cb9cb2986d4a8f23e0 is 50, key is test_row_0/C:col10/1732758166049/Put/seqid=0 2024-11-28T01:42:46,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742344_1520 (size=12001) 2024-11-28T01:42:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T01:42:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758226862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758226872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758226873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758226873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758226873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9464c6d3a2a447cb9cb2986d4a8f23e0 2024-11-28T01:42:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/da0392a3cd2947b5add6885e62fdc45e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/da0392a3cd2947b5add6885e62fdc45e 2024-11-28T01:42:47,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/da0392a3cd2947b5add6885e62fdc45e, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T01:42:47,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/249a9ac6aeaa4f69a347a25142ba6ded as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/249a9ac6aeaa4f69a347a25142ba6ded 2024-11-28T01:42:47,025 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/249a9ac6aeaa4f69a347a25142ba6ded, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T01:42:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9464c6d3a2a447cb9cb2986d4a8f23e0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9464c6d3a2a447cb9cb2986d4a8f23e0 2024-11-28T01:42:47,029 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9464c6d3a2a447cb9cb2986d4a8f23e0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T01:42:47,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 9f37285715eee60df10bafa1eb0d60a0 in 868ms, sequenceid=33, compaction requested=false 2024-11-28T01:42:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-28T01:42:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-28T01:42:47,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-28T01:42:47,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0260 sec 2024-11-28T01:42:47,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0340 sec 2024-11-28T01:42:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T01:42:47,143 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:47,143 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-28T01:42:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T01:42:47,145 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:47,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:47,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T01:42:47,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-28T01:42:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:47,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:42:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:47,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/71dabf7d37204a61bd5fec2035d3976a is 50, key is test_row_0/A:col10/1732758166209/Put/seqid=0 2024-11-28T01:42:47,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742345_1521 (size=12001) 2024-11-28T01:42:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:47,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758227416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758227417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758227419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758227420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758227420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T01:42:47,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758227523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758227542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758227542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758227542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758227542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,709 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/71dabf7d37204a61bd5fec2035d3976a 2024-11-28T01:42:47,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/8dad4a15849141d3a907d63c31fb9227 is 50, key is test_row_0/B:col10/1732758166209/Put/seqid=0 2024-11-28T01:42:47,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742346_1522 (size=12001) 2024-11-28T01:42:47,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758227743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758227762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758227762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758227762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758227763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T01:42:48,019 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T01:42:48,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758228063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758228072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758228072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758228082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758228083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,121 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/8dad4a15849141d3a907d63c31fb9227 2024-11-28T01:42:48,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/72a3c8567b024fa4a6ea95743922bffa is 50, key is test_row_0/C:col10/1732758166209/Put/seqid=0 2024-11-28T01:42:48,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742347_1523 (size=12001) 2024-11-28T01:42:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T01:42:48,533 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/72a3c8567b024fa4a6ea95743922bffa 2024-11-28T01:42:48,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/71dabf7d37204a61bd5fec2035d3976a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71dabf7d37204a61bd5fec2035d3976a 2024-11-28T01:42:48,541 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71dabf7d37204a61bd5fec2035d3976a, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T01:42:48,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/8dad4a15849141d3a907d63c31fb9227 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8dad4a15849141d3a907d63c31fb9227 2024-11-28T01:42:48,547 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8dad4a15849141d3a907d63c31fb9227, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T01:42:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/72a3c8567b024fa4a6ea95743922bffa as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/72a3c8567b024fa4a6ea95743922bffa 2024-11-28T01:42:48,552 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/72a3c8567b024fa4a6ea95743922bffa, entries=150, sequenceid=47, filesize=11.7 K 2024-11-28T01:42:48,553 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 9f37285715eee60df10bafa1eb0d60a0 in 1254ms, sequenceid=47, compaction requested=true 2024-11-28T01:42:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:48,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-28T01:42:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-28T01:42:48,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-28T01:42:48,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-28T01:42:48,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4140 sec 2024-11-28T01:42:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:48,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:42:48,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:48,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:48,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:48,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:48,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:48,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:48,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/a4df3ef860b540d2ab7730d13c0e45cb is 50, key is test_row_0/A:col10/1732758168586/Put/seqid=0 2024-11-28T01:42:48,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742348_1524 (size=14341) 2024-11-28T01:42:48,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/a4df3ef860b540d2ab7730d13c0e45cb 2024-11-28T01:42:48,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/4380900727174776884304d8892ad842 is 50, key is test_row_0/B:col10/1732758168586/Put/seqid=0 2024-11-28T01:42:48,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758228611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758228619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758228619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758228619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742349_1525 (size=12001) 2024-11-28T01:42:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758228621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758228732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758228733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758228733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758228733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758228742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758228943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758228943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758228943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758228943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:48,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758228962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/4380900727174776884304d8892ad842 2024-11-28T01:42:49,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/59d3e7f91e474bf3ba1d795751f09971 is 50, key is test_row_0/C:col10/1732758168586/Put/seqid=0 2024-11-28T01:42:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742350_1526 (size=12001) 2024-11-28T01:42:49,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/59d3e7f91e474bf3ba1d795751f09971 2024-11-28T01:42:49,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/a4df3ef860b540d2ab7730d13c0e45cb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a4df3ef860b540d2ab7730d13c0e45cb 2024-11-28T01:42:49,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a4df3ef860b540d2ab7730d13c0e45cb, entries=200, sequenceid=66, filesize=14.0 K 2024-11-28T01:42:49,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/4380900727174776884304d8892ad842 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4380900727174776884304d8892ad842 2024-11-28T01:42:49,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4380900727174776884304d8892ad842, entries=150, sequenceid=66, filesize=11.7 K 2024-11-28T01:42:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/59d3e7f91e474bf3ba1d795751f09971 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/59d3e7f91e474bf3ba1d795751f09971 2024-11-28T01:42:49,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/59d3e7f91e474bf3ba1d795751f09971, entries=150, sequenceid=66, filesize=11.7 K 2024-11-28T01:42:49,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 9f37285715eee60df10bafa1eb0d60a0 in 477ms, sequenceid=66, compaction requested=true 2024-11-28T01:42:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:49,065 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:49,066 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:49,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:49,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:49,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:49,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:49,067 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:49,067 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:42:49,067 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,067 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/0db253bec1914cef99c6d3c4158fb62d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/249a9ac6aeaa4f69a347a25142ba6ded, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8dad4a15849141d3a907d63c31fb9227, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4380900727174776884304d8892ad842] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=46.9 K 2024-11-28T01:42:49,068 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:49,068 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:42:49,068 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,068 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/bcff672b7e0e41e68417e9b2fd9fb52e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/da0392a3cd2947b5add6885e62fdc45e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71dabf7d37204a61bd5fec2035d3976a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a4df3ef860b540d2ab7730d13c0e45cb] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=49.2 K 2024-11-28T01:42:49,068 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcff672b7e0e41e68417e9b2fd9fb52e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758166014 2024-11-28T01:42:49,068 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db253bec1914cef99c6d3c4158fb62d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758166014 2024-11-28T01:42:49,069 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0392a3cd2947b5add6885e62fdc45e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732758166049 2024-11-28T01:42:49,069 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 249a9ac6aeaa4f69a347a25142ba6ded, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732758166049 2024-11-28T01:42:49,069 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71dabf7d37204a61bd5fec2035d3976a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732758166209 2024-11-28T01:42:49,069 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dad4a15849141d3a907d63c31fb9227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732758166209 2024-11-28T01:42:49,069 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4df3ef860b540d2ab7730d13c0e45cb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732758167414 2024-11-28T01:42:49,070 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4380900727174776884304d8892ad842, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732758167414 2024-11-28T01:42:49,085 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:49,085 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/72fed8bc752843f2a9d6bd99070fd073 is 50, key is test_row_0/B:col10/1732758168586/Put/seqid=0 2024-11-28T01:42:49,093 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:49,093 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/b840ed34a6284a3ba8f9908f6c452837 is 50, key is test_row_0/A:col10/1732758168586/Put/seqid=0 2024-11-28T01:42:49,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742351_1527 (size=12139) 2024-11-28T01:42:49,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742352_1528 (size=12139) 2024-11-28T01:42:49,109 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/b840ed34a6284a3ba8f9908f6c452837 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b840ed34a6284a3ba8f9908f6c452837 2024-11-28T01:42:49,116 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into b840ed34a6284a3ba8f9908f6c452837(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:49,117 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:49,117 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=12, startTime=1732758169065; duration=0sec 2024-11-28T01:42:49,117 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:49,117 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:42:49,117 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:49,118 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:49,118 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:42:49,118 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,118 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/964cc5b9b4a54e09bcd98dad4211db0b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9464c6d3a2a447cb9cb2986d4a8f23e0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/72a3c8567b024fa4a6ea95743922bffa, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/59d3e7f91e474bf3ba1d795751f09971] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=46.9 K 2024-11-28T01:42:49,119 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 964cc5b9b4a54e09bcd98dad4211db0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758166014 2024-11-28T01:42:49,119 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9464c6d3a2a447cb9cb2986d4a8f23e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732758166049 2024-11-28T01:42:49,120 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72a3c8567b024fa4a6ea95743922bffa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732758166209 2024-11-28T01:42:49,120 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59d3e7f91e474bf3ba1d795751f09971, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732758167414 2024-11-28T01:42:49,130 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:49,131 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/66259ca9254a49faa48913bf34c6e2b1 is 50, key is test_row_0/C:col10/1732758168586/Put/seqid=0 2024-11-28T01:42:49,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742353_1529 (size=12139) 2024-11-28T01:42:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:49,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:42:49,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:49,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:49,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:49,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:49,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:49,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:49,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/d0d4cf4c90fa4deaaad30f853ff4ee47 is 50, key is test_row_0/A:col10/1732758169277/Put/seqid=0 2024-11-28T01:42:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T01:42:49,293 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:49,293 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-28T01:42:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T01:42:49,295 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:49,296 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:49,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:49,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742354_1530 (size=14341) 2024-11-28T01:42:49,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758229304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758229318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758229318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758229319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758229319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T01:42:49,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758229422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758229433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T01:42:49,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758229442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758229442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758229442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,506 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/72fed8bc752843f2a9d6bd99070fd073 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/72fed8bc752843f2a9d6bd99070fd073 2024-11-28T01:42:49,511 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into 72fed8bc752843f2a9d6bd99070fd073(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:49,511 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:49,511 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=12, startTime=1732758169065; duration=0sec 2024-11-28T01:42:49,511 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:49,511 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:42:49,541 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/66259ca9254a49faa48913bf34c6e2b1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/66259ca9254a49faa48913bf34c6e2b1 2024-11-28T01:42:49,545 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into 66259ca9254a49faa48913bf34c6e2b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:49,545 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:49,545 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=12, startTime=1732758169066; duration=0sec 2024-11-28T01:42:49,545 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:49,545 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:42:49,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T01:42:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T01:42:49,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758229643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758229643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758229662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758229662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758229662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/d0d4cf4c90fa4deaaad30f853ff4ee47 2024-11-28T01:42:49,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/398fe6f0936b4c86b95004b36ec9a5fe is 50, key is test_row_0/B:col10/1732758169277/Put/seqid=0 2024-11-28T01:42:49,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742355_1531 (size=12001) 2024-11-28T01:42:49,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T01:42:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T01:42:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:49,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T01:42:49,953 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:42:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758229962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758229962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758229972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758229983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758229983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:49,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T01:42:50,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:50,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:50,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:50,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:50,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:50,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/398fe6f0936b4c86b95004b36ec9a5fe 2024-11-28T01:42:50,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/e132526f07a8456cb8705a93c952e22c is 50, key is test_row_0/C:col10/1732758169277/Put/seqid=0 2024-11-28T01:42:50,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742356_1532 (size=12001) 2024-11-28T01:42:50,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/e132526f07a8456cb8705a93c952e22c 2024-11-28T01:42:50,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/d0d4cf4c90fa4deaaad30f853ff4ee47 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d0d4cf4c90fa4deaaad30f853ff4ee47 2024-11-28T01:42:50,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d0d4cf4c90fa4deaaad30f853ff4ee47, entries=200, sequenceid=83, filesize=14.0 K 2024-11-28T01:42:50,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/398fe6f0936b4c86b95004b36ec9a5fe as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/398fe6f0936b4c86b95004b36ec9a5fe 2024-11-28T01:42:50,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/398fe6f0936b4c86b95004b36ec9a5fe, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T01:42:50,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/e132526f07a8456cb8705a93c952e22c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e132526f07a8456cb8705a93c952e22c 2024-11-28T01:42:50,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e132526f07a8456cb8705a93c952e22c, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T01:42:50,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 9f37285715eee60df10bafa1eb0d60a0 in 862ms, sequenceid=83, compaction requested=false 2024-11-28T01:42:50,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:50,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T01:42:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:50,214 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T01:42:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:50,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:50,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:50,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/fb9db6fe9eb24be08c10e9efaaac8e55 is 50, key is test_row_0/A:col10/1732758169316/Put/seqid=0 2024-11-28T01:42:50,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742357_1533 (size=12001) 2024-11-28T01:42:50,223 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/fb9db6fe9eb24be08c10e9efaaac8e55 2024-11-28T01:42:50,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/22f35b9b9ab64176af645962f9463ea4 is 50, key is test_row_0/B:col10/1732758169316/Put/seqid=0 2024-11-28T01:42:50,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742358_1534 (size=12001) 2024-11-28T01:42:50,236 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/22f35b9b9ab64176af645962f9463ea4 2024-11-28T01:42:50,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/7b9b7d4a00bf46f48f17691143373f0b is 50, key is test_row_0/C:col10/1732758169316/Put/seqid=0 2024-11-28T01:42:50,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742359_1535 (size=12001) 2024-11-28T01:42:50,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T01:42:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:50,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:50,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758230515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758230516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758230517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758230517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758230518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758230623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758230623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758230642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758230642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758230643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,654 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/7b9b7d4a00bf46f48f17691143373f0b 2024-11-28T01:42:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/fb9db6fe9eb24be08c10e9efaaac8e55 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/fb9db6fe9eb24be08c10e9efaaac8e55 2024-11-28T01:42:50,661 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/fb9db6fe9eb24be08c10e9efaaac8e55, entries=150, sequenceid=102, filesize=11.7 K 2024-11-28T01:42:50,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/22f35b9b9ab64176af645962f9463ea4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/22f35b9b9ab64176af645962f9463ea4 2024-11-28T01:42:50,665 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/22f35b9b9ab64176af645962f9463ea4, entries=150, sequenceid=102, filesize=11.7 K 2024-11-28T01:42:50,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/7b9b7d4a00bf46f48f17691143373f0b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/7b9b7d4a00bf46f48f17691143373f0b 2024-11-28T01:42:50,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/7b9b7d4a00bf46f48f17691143373f0b, entries=150, sequenceid=102, filesize=11.7 K 2024-11-28T01:42:50,669 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 9f37285715eee60df10bafa1eb0d60a0 in 455ms, sequenceid=102, compaction requested=true 2024-11-28T01:42:50,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:50,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:50,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-28T01:42:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-28T01:42:50,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-28T01:42:50,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3740 sec 2024-11-28T01:42:50,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3790 sec 2024-11-28T01:42:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:50,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T01:42:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:50,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:50,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/7496f887553a4d6f9f4e74ac5c4c8fdf is 50, key is test_row_0/A:col10/1732758170514/Put/seqid=0 2024-11-28T01:42:50,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742360_1536 (size=14341) 2024-11-28T01:42:50,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758230876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758230877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758230879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758230885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:50,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758230888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:50,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758231002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758231002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758231002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758231013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758231013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758231223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758231223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758231223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758231232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758231232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/7496f887553a4d6f9f4e74ac5c4c8fdf 2024-11-28T01:42:51,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/70bf5a23acd8439da54c0253eb580bc8 is 50, key is test_row_0/B:col10/1732758170514/Put/seqid=0 2024-11-28T01:42:51,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742361_1537 (size=12001) 2024-11-28T01:42:51,403 WARN [regionserver/9df3f25f5867:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-28T01:42:51,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/bcff672b7e0e41e68417e9b2fd9fb52e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/da0392a3cd2947b5add6885e62fdc45e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71dabf7d37204a61bd5fec2035d3976a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a4df3ef860b540d2ab7730d13c0e45cb] to archive 2024-11-28T01:42:51,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:51,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/bcff672b7e0e41e68417e9b2fd9fb52e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/bcff672b7e0e41e68417e9b2fd9fb52e 2024-11-28T01:42:51,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/da0392a3cd2947b5add6885e62fdc45e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/da0392a3cd2947b5add6885e62fdc45e 2024-11-28T01:42:51,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71dabf7d37204a61bd5fec2035d3976a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71dabf7d37204a61bd5fec2035d3976a 2024-11-28T01:42:51,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a4df3ef860b540d2ab7730d13c0e45cb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a4df3ef860b540d2ab7730d13c0e45cb 2024-11-28T01:42:51,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/0db253bec1914cef99c6d3c4158fb62d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/249a9ac6aeaa4f69a347a25142ba6ded, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8dad4a15849141d3a907d63c31fb9227, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4380900727174776884304d8892ad842] to archive 2024-11-28T01:42:51,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:51,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/0db253bec1914cef99c6d3c4158fb62d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/0db253bec1914cef99c6d3c4158fb62d 2024-11-28T01:42:51,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/249a9ac6aeaa4f69a347a25142ba6ded to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/249a9ac6aeaa4f69a347a25142ba6ded 2024-11-28T01:42:51,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8dad4a15849141d3a907d63c31fb9227 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8dad4a15849141d3a907d63c31fb9227 2024-11-28T01:42:51,440 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4380900727174776884304d8892ad842 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4380900727174776884304d8892ad842 2024-11-28T01:42:51,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/964cc5b9b4a54e09bcd98dad4211db0b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9464c6d3a2a447cb9cb2986d4a8f23e0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/72a3c8567b024fa4a6ea95743922bffa, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/59d3e7f91e474bf3ba1d795751f09971] to archive 2024-11-28T01:42:51,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:42:51,447 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/964cc5b9b4a54e09bcd98dad4211db0b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/964cc5b9b4a54e09bcd98dad4211db0b 2024-11-28T01:42:51,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9464c6d3a2a447cb9cb2986d4a8f23e0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9464c6d3a2a447cb9cb2986d4a8f23e0 2024-11-28T01:42:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T01:42:51,453 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:51,454 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-28T01:42:51,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/72a3c8567b024fa4a6ea95743922bffa to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/72a3c8567b024fa4a6ea95743922bffa 2024-11-28T01:42:51,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:51,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/59d3e7f91e474bf3ba1d795751f09971 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/59d3e7f91e474bf3ba1d795751f09971 2024-11-28T01:42:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T01:42:51,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:51,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:51,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758231552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758231552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758231553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758231553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758231553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T01:42:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:51,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T01:42:51,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:51,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:51,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:51,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/70bf5a23acd8439da54c0253eb580bc8 2024-11-28T01:42:51,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/118ebdab152a4635af91d29146e95d7b is 50, key is test_row_0/C:col10/1732758170514/Put/seqid=0 2024-11-28T01:42:51,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742362_1538 (size=12001) 2024-11-28T01:42:51,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T01:42:51,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:51,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:51,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T01:42:51,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T01:42:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:51,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:52,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T01:42:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:52,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758232072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758232072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758232073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T01:42:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758232073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:52,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758232073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:52,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/118ebdab152a4635af91d29146e95d7b 2024-11-28T01:42:52,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/7496f887553a4d6f9f4e74ac5c4c8fdf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7496f887553a4d6f9f4e74ac5c4c8fdf 2024-11-28T01:42:52,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7496f887553a4d6f9f4e74ac5c4c8fdf, entries=200, sequenceid=118, filesize=14.0 K 2024-11-28T01:42:52,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/70bf5a23acd8439da54c0253eb580bc8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/70bf5a23acd8439da54c0253eb580bc8 2024-11-28T01:42:52,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/70bf5a23acd8439da54c0253eb580bc8, entries=150, sequenceid=118, filesize=11.7 K 2024-11-28T01:42:52,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/118ebdab152a4635af91d29146e95d7b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/118ebdab152a4635af91d29146e95d7b 2024-11-28T01:42:52,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/118ebdab152a4635af91d29146e95d7b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-28T01:42:52,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 9f37285715eee60df10bafa1eb0d60a0 in 1296ms, sequenceid=118, compaction requested=true 2024-11-28T01:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:52,134 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:52,134 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:52,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:52,136 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52822 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:52,136 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:42:52,136 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:52,136 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b840ed34a6284a3ba8f9908f6c452837, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d0d4cf4c90fa4deaaad30f853ff4ee47, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/fb9db6fe9eb24be08c10e9efaaac8e55, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7496f887553a4d6f9f4e74ac5c4c8fdf] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=51.6 K 2024-11-28T01:42:52,137 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:52,137 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:42:52,137 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:52,137 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/72fed8bc752843f2a9d6bd99070fd073, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/398fe6f0936b4c86b95004b36ec9a5fe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/22f35b9b9ab64176af645962f9463ea4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/70bf5a23acd8439da54c0253eb580bc8] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=47.0 K 2024-11-28T01:42:52,137 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b840ed34a6284a3ba8f9908f6c452837, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732758167414 2024-11-28T01:42:52,137 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 72fed8bc752843f2a9d6bd99070fd073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732758167414 2024-11-28T01:42:52,137 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d4cf4c90fa4deaaad30f853ff4ee47, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758168614 2024-11-28T01:42:52,137 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 398fe6f0936b4c86b95004b36ec9a5fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758168614 2024-11-28T01:42:52,137 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 22f35b9b9ab64176af645962f9463ea4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732758169316 2024-11-28T01:42:52,138 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb9db6fe9eb24be08c10e9efaaac8e55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732758169316 2024-11-28T01:42:52,138 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7496f887553a4d6f9f4e74ac5c4c8fdf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732758170504 2024-11-28T01:42:52,138 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 70bf5a23acd8439da54c0253eb580bc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732758170514 2024-11-28T01:42:52,153 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#451 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:52,153 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/7cd48122a2864d2fbbde5fef443e88e4 is 50, key is test_row_0/A:col10/1732758170514/Put/seqid=0 2024-11-28T01:42:52,155 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#452 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:52,155 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/900d8e2d7cea49c59e6d8580cbe92f85 is 50, key is test_row_0/B:col10/1732758170514/Put/seqid=0 2024-11-28T01:42:52,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742363_1539 (size=12139) 2024-11-28T01:42:52,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742364_1540 (size=12139) 2024-11-28T01:42:52,165 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/7cd48122a2864d2fbbde5fef443e88e4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7cd48122a2864d2fbbde5fef443e88e4 2024-11-28T01:42:52,171 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into 7cd48122a2864d2fbbde5fef443e88e4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:52,171 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:52,171 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=12, startTime=1732758172134; duration=0sec 2024-11-28T01:42:52,171 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:52,171 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:42:52,171 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:42:52,172 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:42:52,172 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:42:52,172 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:52,172 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/66259ca9254a49faa48913bf34c6e2b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e132526f07a8456cb8705a93c952e22c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/7b9b7d4a00bf46f48f17691143373f0b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/118ebdab152a4635af91d29146e95d7b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=47.0 K 2024-11-28T01:42:52,172 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66259ca9254a49faa48913bf34c6e2b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732758167414 2024-11-28T01:42:52,172 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e132526f07a8456cb8705a93c952e22c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758168614 2024-11-28T01:42:52,173 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b9b7d4a00bf46f48f17691143373f0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1732758169316 2024-11-28T01:42:52,173 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 118ebdab152a4635af91d29146e95d7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732758170514 2024-11-28T01:42:52,181 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#453 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:52,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/44b23a41b6a743f18e3d59cd165c2c0d is 50, key is test_row_0/C:col10/1732758170514/Put/seqid=0 2024-11-28T01:42:52,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742365_1541 (size=12139) 2024-11-28T01:42:52,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T01:42:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:52,225 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T01:42:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:52,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/f360a0ff5cde4a5f9243342180f1affb is 50, key is test_row_0/A:col10/1732758170884/Put/seqid=0 2024-11-28T01:42:52,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742366_1542 (size=12151) 2024-11-28T01:42:52,566 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/900d8e2d7cea49c59e6d8580cbe92f85 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/900d8e2d7cea49c59e6d8580cbe92f85 2024-11-28T01:42:52,570 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into 900d8e2d7cea49c59e6d8580cbe92f85(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:52,570 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:52,570 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=12, startTime=1732758172134; duration=0sec 2024-11-28T01:42:52,570 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:52,570 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:42:52,590 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/44b23a41b6a743f18e3d59cd165c2c0d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/44b23a41b6a743f18e3d59cd165c2c0d 2024-11-28T01:42:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T01:42:52,594 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into 44b23a41b6a743f18e3d59cd165c2c0d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:52,595 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:52,595 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=12, startTime=1732758172134; duration=0sec 2024-11-28T01:42:52,595 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:52,595 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:42:52,633 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/f360a0ff5cde4a5f9243342180f1affb 2024-11-28T01:42:52,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/ae02c2eda1b14787884f5b96623a87ec is 50, key is test_row_0/B:col10/1732758170884/Put/seqid=0 2024-11-28T01:42:52,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742367_1543 (size=12151) 2024-11-28T01:42:52,644 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/ae02c2eda1b14787884f5b96623a87ec 2024-11-28T01:42:52,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/5e89e390ac0f42c3b5c5099ecbeb7153 is 50, key is test_row_0/C:col10/1732758170884/Put/seqid=0 2024-11-28T01:42:52,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742368_1544 (size=12151) 2024-11-28T01:42:53,057 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/5e89e390ac0f42c3b5c5099ecbeb7153 2024-11-28T01:42:53,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/f360a0ff5cde4a5f9243342180f1affb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/f360a0ff5cde4a5f9243342180f1affb 2024-11-28T01:42:53,065 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/f360a0ff5cde4a5f9243342180f1affb, entries=150, sequenceid=135, filesize=11.9 K 2024-11-28T01:42:53,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/ae02c2eda1b14787884f5b96623a87ec as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/ae02c2eda1b14787884f5b96623a87ec 2024-11-28T01:42:53,069 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/ae02c2eda1b14787884f5b96623a87ec, entries=150, sequenceid=135, filesize=11.9 K 2024-11-28T01:42:53,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/5e89e390ac0f42c3b5c5099ecbeb7153 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/5e89e390ac0f42c3b5c5099ecbeb7153 2024-11-28T01:42:53,073 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/5e89e390ac0f42c3b5c5099ecbeb7153, entries=150, sequenceid=135, filesize=11.9 K 2024-11-28T01:42:53,074 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 9f37285715eee60df10bafa1eb0d60a0 in 849ms, sequenceid=135, compaction requested=false 2024-11-28T01:42:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-28T01:42:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-28T01:42:53,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-28T01:42:53,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-28T01:42:53,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6240 sec 2024-11-28T01:42:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:53,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:42:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:53,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/78f28ab1a85b4d68b52cbff1f99b6619 is 50, key is test_row_0/A:col10/1732758173097/Put/seqid=0 2024-11-28T01:42:53,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742369_1545 (size=14541) 2024-11-28T01:42:53,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/78f28ab1a85b4d68b52cbff1f99b6619 2024-11-28T01:42:53,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/369d3189773f4afc8bff88072b6a0386 is 50, key is test_row_0/B:col10/1732758173097/Put/seqid=0 2024-11-28T01:42:53,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742370_1546 (size=12151) 2024-11-28T01:42:53,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732758173129, max timestamp=1732758173149Num uniques 859; 2024-11-28T01:42:53,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T01:42:53,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732758173129, max timestamp=1732758173149Num uniques 859; 2024-11-28T01:42:53,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T01:42:53,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732758173129, max timestamp=1732758173149Num uniques 859; 2024-11-28T01:42:53,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T01:42:53,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758233177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758233193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758233193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758233195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758233197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758233302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758233312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758233312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758233323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758233323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/369d3189773f4afc8bff88072b6a0386 2024-11-28T01:42:53,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/020dc3875fc3456fa7d5a7ec76bd5b7b is 50, key is test_row_0/C:col10/1732758173097/Put/seqid=0 2024-11-28T01:42:53,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742371_1547 (size=12151) 2024-11-28T01:42:53,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758233523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758233533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758233533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758233542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758233542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T01:42:53,602 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:53,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-28T01:42:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T01:42:53,605 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:53,605 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:53,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T01:42:53,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T01:42:53,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:53,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:53,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758233842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758233852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758233852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758233863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758233863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:53,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:53,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T01:42:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T01:42:53,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/020dc3875fc3456fa7d5a7ec76bd5b7b 2024-11-28T01:42:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/78f28ab1a85b4d68b52cbff1f99b6619 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/78f28ab1a85b4d68b52cbff1f99b6619 2024-11-28T01:42:53,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/78f28ab1a85b4d68b52cbff1f99b6619, entries=200, sequenceid=147, filesize=14.2 K 2024-11-28T01:42:53,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/369d3189773f4afc8bff88072b6a0386 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/369d3189773f4afc8bff88072b6a0386 2024-11-28T01:42:53,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/369d3189773f4afc8bff88072b6a0386, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T01:42:53,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/020dc3875fc3456fa7d5a7ec76bd5b7b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020dc3875fc3456fa7d5a7ec76bd5b7b 2024-11-28T01:42:53,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020dc3875fc3456fa7d5a7ec76bd5b7b, entries=150, sequenceid=147, filesize=11.9 K 2024-11-28T01:42:53,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 9f37285715eee60df10bafa1eb0d60a0 in 850ms, sequenceid=147, compaction requested=true 2024-11-28T01:42:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:53,950 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:53,950 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:53,952 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:53,952 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:42:53,952 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,952 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7cd48122a2864d2fbbde5fef443e88e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/f360a0ff5cde4a5f9243342180f1affb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/78f28ab1a85b4d68b52cbff1f99b6619] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=37.9 K 2024-11-28T01:42:53,955 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:53,955 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:42:53,955 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cd48122a2864d2fbbde5fef443e88e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732758170514 2024-11-28T01:42:53,955 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,956 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/900d8e2d7cea49c59e6d8580cbe92f85, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/ae02c2eda1b14787884f5b96623a87ec, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/369d3189773f4afc8bff88072b6a0386] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=35.6 K 2024-11-28T01:42:53,956 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f360a0ff5cde4a5f9243342180f1affb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732758170876 2024-11-28T01:42:53,956 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 900d8e2d7cea49c59e6d8580cbe92f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732758170514 2024-11-28T01:42:53,956 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f28ab1a85b4d68b52cbff1f99b6619, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732758173096 2024-11-28T01:42:53,956 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ae02c2eda1b14787884f5b96623a87ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732758170876 2024-11-28T01:42:53,957 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 369d3189773f4afc8bff88072b6a0386, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732758173096 2024-11-28T01:42:53,963 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:53,963 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:53,964 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/095ee18a3c13472189e379f399cb0d43 is 50, key is test_row_0/A:col10/1732758173097/Put/seqid=0 2024-11-28T01:42:53,964 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/c94ea028846f4e398414caecf6bc3762 is 50, key is test_row_0/B:col10/1732758173097/Put/seqid=0 2024-11-28T01:42:53,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742372_1548 (size=12391) 2024-11-28T01:42:53,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742373_1549 (size=12391) 2024-11-28T01:42:53,972 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/c94ea028846f4e398414caecf6bc3762 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c94ea028846f4e398414caecf6bc3762 2024-11-28T01:42:53,972 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/095ee18a3c13472189e379f399cb0d43 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/095ee18a3c13472189e379f399cb0d43 2024-11-28T01:42:53,976 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into 095ee18a3c13472189e379f399cb0d43(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:53,976 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:53,976 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=13, startTime=1732758173950; duration=0sec 2024-11-28T01:42:53,976 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:53,976 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:42:53,976 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:53,977 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into c94ea028846f4e398414caecf6bc3762(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:53,977 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:53,977 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=13, startTime=1732758173950; duration=0sec 2024-11-28T01:42:53,977 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:53,977 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:42:53,978 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:53,978 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:42:53,978 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:53,978 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/44b23a41b6a743f18e3d59cd165c2c0d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/5e89e390ac0f42c3b5c5099ecbeb7153, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020dc3875fc3456fa7d5a7ec76bd5b7b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=35.6 K 2024-11-28T01:42:53,978 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44b23a41b6a743f18e3d59cd165c2c0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732758170514 2024-11-28T01:42:53,979 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e89e390ac0f42c3b5c5099ecbeb7153, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732758170876 2024-11-28T01:42:53,979 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 020dc3875fc3456fa7d5a7ec76bd5b7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732758173096 2024-11-28T01:42:53,986 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:53,986 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/f9bf8e3df2fb4a56985028d265ab3d07 is 50, key is test_row_0/C:col10/1732758173097/Put/seqid=0 2024-11-28T01:42:53,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742374_1550 (size=12391) 2024-11-28T01:42:54,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T01:42:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:54,064 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-28T01:42:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/8cd2de9144a24613b1d5711ad68f19e3 is 50, key is test_row_0/A:col10/1732758173193/Put/seqid=0 2024-11-28T01:42:54,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742375_1551 (size=12151) 2024-11-28T01:42:54,073 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/8cd2de9144a24613b1d5711ad68f19e3 2024-11-28T01:42:54,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/4fc44dd67d614751acc14204bd15b560 is 50, key is test_row_0/B:col10/1732758173193/Put/seqid=0 2024-11-28T01:42:54,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742376_1552 (size=12151) 2024-11-28T01:42:54,083 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/4fc44dd67d614751acc14204bd15b560 2024-11-28T01:42:54,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/2581ad9c073b49808292aaf9c8aa0088 is 50, key is test_row_0/C:col10/1732758173193/Put/seqid=0 2024-11-28T01:42:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742377_1553 (size=12151) 2024-11-28T01:42:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T01:42:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:54,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:54,394 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/f9bf8e3df2fb4a56985028d265ab3d07 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f9bf8e3df2fb4a56985028d265ab3d07 2024-11-28T01:42:54,399 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into f9bf8e3df2fb4a56985028d265ab3d07(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:54,400 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:54,400 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=13, startTime=1732758173950; duration=0sec 2024-11-28T01:42:54,400 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:54,400 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:42:54,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758234387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758234392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758234394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758234396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758234398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,493 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/2581ad9c073b49808292aaf9c8aa0088 2024-11-28T01:42:54,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/8cd2de9144a24613b1d5711ad68f19e3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/8cd2de9144a24613b1d5711ad68f19e3 2024-11-28T01:42:54,500 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/8cd2de9144a24613b1d5711ad68f19e3, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T01:42:54,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/4fc44dd67d614751acc14204bd15b560 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4fc44dd67d614751acc14204bd15b560 2024-11-28T01:42:54,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4fc44dd67d614751acc14204bd15b560, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T01:42:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/2581ad9c073b49808292aaf9c8aa0088 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2581ad9c073b49808292aaf9c8aa0088 2024-11-28T01:42:54,509 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2581ad9c073b49808292aaf9c8aa0088, entries=150, sequenceid=183, filesize=11.9 K 2024-11-28T01:42:54,510 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 9f37285715eee60df10bafa1eb0d60a0 in 446ms, sequenceid=183, compaction requested=false 2024-11-28T01:42:54,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:54,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:54,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-28T01:42:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-28T01:42:54,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-28T01:42:54,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 907 msec 2024-11-28T01:42:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:54,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T01:42:54,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:54,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:54,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:54,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:54,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:54,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:54,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 912 msec 2024-11-28T01:42:54,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/95f47208dc43489c907e91406c816212 is 50, key is test_row_0/A:col10/1732758174393/Put/seqid=0 2024-11-28T01:42:54,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742378_1554 (size=14541) 2024-11-28T01:42:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758234557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758234558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732758174546, max timestamp=1732758174566Num uniques 859; 2024-11-28T01:42:54,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T01:42:54,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732758174546, max timestamp=1732758174566Num uniques 859; 2024-11-28T01:42:54,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T01:42:54,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732758174546, max timestamp=1732758174566Num uniques 859; 2024-11-28T01:42:54,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T01:42:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758234621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758234623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758234624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758234672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758234683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T01:42:54,743 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:54,743 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-28T01:42:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T01:42:54,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:54,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:54,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:54,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758234732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758234742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758234753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T01:42:54,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758234882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T01:42:54,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:54,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:54,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:54,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758234902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/95f47208dc43489c907e91406c816212 2024-11-28T01:42:54,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/797a857304844fb8bfbec6f807e9198d is 50, key is test_row_0/B:col10/1732758174393/Put/seqid=0 2024-11-28T01:42:54,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742379_1555 (size=12151) 2024-11-28T01:42:54,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758234953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758234962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758234972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T01:42:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T01:42:55,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T01:42:55,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:55,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758235202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758235223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758235272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758235283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758235293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T01:42:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:55,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/797a857304844fb8bfbec6f807e9198d 2024-11-28T01:42:55,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/569892c3fe4d4bc8b222618420c26f2a is 50, key is test_row_0/C:col10/1732758174393/Put/seqid=0 2024-11-28T01:42:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T01:42:55,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742380_1556 (size=12151) 2024-11-28T01:42:55,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/569892c3fe4d4bc8b222618420c26f2a 2024-11-28T01:42:55,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/95f47208dc43489c907e91406c816212 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/95f47208dc43489c907e91406c816212 2024-11-28T01:42:55,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/95f47208dc43489c907e91406c816212, entries=200, sequenceid=195, filesize=14.2 K 2024-11-28T01:42:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/797a857304844fb8bfbec6f807e9198d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/797a857304844fb8bfbec6f807e9198d 2024-11-28T01:42:55,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/797a857304844fb8bfbec6f807e9198d, entries=150, sequenceid=195, filesize=11.9 K 2024-11-28T01:42:55,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/569892c3fe4d4bc8b222618420c26f2a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/569892c3fe4d4bc8b222618420c26f2a 2024-11-28T01:42:55,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/569892c3fe4d4bc8b222618420c26f2a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-28T01:42:55,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 9f37285715eee60df10bafa1eb0d60a0 in 881ms, sequenceid=195, compaction requested=true 2024-11-28T01:42:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:55,398 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:55,398 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:55,400 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:55,400 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:55,400 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:42:55,400 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:42:55,400 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,400 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,400 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/095ee18a3c13472189e379f399cb0d43, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/8cd2de9144a24613b1d5711ad68f19e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/95f47208dc43489c907e91406c816212] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=38.2 K 2024-11-28T01:42:55,400 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c94ea028846f4e398414caecf6bc3762, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4fc44dd67d614751acc14204bd15b560, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/797a857304844fb8bfbec6f807e9198d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=35.8 K 2024-11-28T01:42:55,400 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095ee18a3c13472189e379f399cb0d43, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732758173096 2024-11-28T01:42:55,400 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c94ea028846f4e398414caecf6bc3762, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732758173096 2024-11-28T01:42:55,401 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cd2de9144a24613b1d5711ad68f19e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732758173191 2024-11-28T01:42:55,401 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fc44dd67d614751acc14204bd15b560, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732758173191 2024-11-28T01:42:55,401 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95f47208dc43489c907e91406c816212, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732758174387 2024-11-28T01:42:55,401 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 797a857304844fb8bfbec6f807e9198d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732758174387 2024-11-28T01:42:55,410 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:55,410 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/6450e77f82a4471ca25ee4216b81e5ff is 50, key is test_row_0/A:col10/1732758174393/Put/seqid=0 2024-11-28T01:42:55,417 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:55,418 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/2834f6a4cf424228a2344b69ed46e2a9 is 50, key is test_row_0/B:col10/1732758174393/Put/seqid=0 2024-11-28T01:42:55,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742381_1557 (size=12493) 2024-11-28T01:42:55,434 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/6450e77f82a4471ca25ee4216b81e5ff as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/6450e77f82a4471ca25ee4216b81e5ff 2024-11-28T01:42:55,444 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into 6450e77f82a4471ca25ee4216b81e5ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:55,444 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:55,444 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=13, startTime=1732758175398; duration=0sec 2024-11-28T01:42:55,444 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:55,444 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:42:55,445 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:55,445 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:55,445 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:42:55,445 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,446 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f9bf8e3df2fb4a56985028d265ab3d07, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2581ad9c073b49808292aaf9c8aa0088, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/569892c3fe4d4bc8b222618420c26f2a] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=35.8 K 2024-11-28T01:42:55,446 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9bf8e3df2fb4a56985028d265ab3d07, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732758173096 2024-11-28T01:42:55,446 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2581ad9c073b49808292aaf9c8aa0088, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732758173191 2024-11-28T01:42:55,446 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 569892c3fe4d4bc8b222618420c26f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732758174387 2024-11-28T01:42:55,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742382_1558 (size=12493) 2024-11-28T01:42:55,454 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:55,455 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/79e5aece83b84c7ebc9d201f08f30356 is 50, key is test_row_0/C:col10/1732758174393/Put/seqid=0 2024-11-28T01:42:55,462 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/2834f6a4cf424228a2344b69ed46e2a9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/2834f6a4cf424228a2344b69ed46e2a9 2024-11-28T01:42:55,473 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into 2834f6a4cf424228a2344b69ed46e2a9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:55,473 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:55,473 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=13, startTime=1732758175398; duration=0sec 2024-11-28T01:42:55,473 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:55,473 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:42:55,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742383_1559 (size=12493) 2024-11-28T01:42:55,479 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/79e5aece83b84c7ebc9d201f08f30356 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/79e5aece83b84c7ebc9d201f08f30356 2024-11-28T01:42:55,484 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into 79e5aece83b84c7ebc9d201f08f30356(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:55,484 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:55,484 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=13, startTime=1732758175398; duration=0sec 2024-11-28T01:42:55,484 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:55,484 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:42:55,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T01:42:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:55,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-28T01:42:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:42:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/04b527a22b734a5fbab0f93701660507 is 50, key is test_row_0/A:col10/1732758174619/Put/seqid=0 2024-11-28T01:42:55,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742384_1560 (size=12151) 2024-11-28T01:42:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:55,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758235763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758235768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758235782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758235802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758235802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T01:42:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758235872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758235883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:55,925 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/04b527a22b734a5fbab0f93701660507 2024-11-28T01:42:55,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/8c9e30c2013a47bd8e8e5eda4f0aa01c is 50, key is test_row_0/B:col10/1732758174619/Put/seqid=0 2024-11-28T01:42:55,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742385_1561 (size=12151) 2024-11-28T01:42:56,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758236092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758236092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,338 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/8c9e30c2013a47bd8e8e5eda4f0aa01c 2024-11-28T01:42:56,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/4431bbcffa634cd6950117b0195df9d9 is 50, key is test_row_0/C:col10/1732758174619/Put/seqid=0 2024-11-28T01:42:56,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742386_1562 (size=12151) 2024-11-28T01:42:56,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758236413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758236413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,755 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/4431bbcffa634cd6950117b0195df9d9 2024-11-28T01:42:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/04b527a22b734a5fbab0f93701660507 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/04b527a22b734a5fbab0f93701660507 2024-11-28T01:42:56,763 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/04b527a22b734a5fbab0f93701660507, entries=150, sequenceid=231, filesize=11.9 K 2024-11-28T01:42:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/8c9e30c2013a47bd8e8e5eda4f0aa01c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8c9e30c2013a47bd8e8e5eda4f0aa01c 2024-11-28T01:42:56,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8c9e30c2013a47bd8e8e5eda4f0aa01c, entries=150, sequenceid=231, filesize=11.9 K 2024-11-28T01:42:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/4431bbcffa634cd6950117b0195df9d9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/4431bbcffa634cd6950117b0195df9d9 2024-11-28T01:42:56,771 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/4431bbcffa634cd6950117b0195df9d9, entries=150, sequenceid=231, filesize=11.9 K 2024-11-28T01:42:56,772 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 9f37285715eee60df10bafa1eb0d60a0 in 1261ms, sequenceid=231, compaction requested=false 2024-11-28T01:42:56,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:56,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:56,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-28T01:42:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-28T01:42:56,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-28T01:42:56,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0280 sec 2024-11-28T01:42:56,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.0340 sec 2024-11-28T01:42:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:56,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:42:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:56,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/527bdceb055240d7b349e0758ecd8149 is 50, key is test_row_0/A:col10/1732758175762/Put/seqid=0 2024-11-28T01:42:56,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742387_1563 (size=14541) 2024-11-28T01:42:56,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/527bdceb055240d7b349e0758ecd8149 2024-11-28T01:42:56,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/b6a7daf66c994330ac079199bfe1325f is 50, key is test_row_0/B:col10/1732758175762/Put/seqid=0 2024-11-28T01:42:56,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742388_1564 (size=12151) 2024-11-28T01:42:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T01:42:56,893 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:56,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-28T01:42:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T01:42:56,896 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:56,897 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:56,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:56,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758236898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758236900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758236913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758236932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758236932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T01:42:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758237022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758237022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758237033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T01:42:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T01:42:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T01:42:57,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/b6a7daf66c994330ac079199bfe1325f 2024-11-28T01:42:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758237242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/d36ba121e2484d41b11043195616be68 is 50, key is test_row_0/C:col10/1732758175762/Put/seqid=0 2024-11-28T01:42:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758237242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742389_1565 (size=12151) 2024-11-28T01:42:57,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758237253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T01:42:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T01:42:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T01:42:57,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758237563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758237563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758237572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:57,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:57,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/d36ba121e2484d41b11043195616be68 2024-11-28T01:42:57,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T01:42:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:57,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/527bdceb055240d7b349e0758ecd8149 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/527bdceb055240d7b349e0758ecd8149 2024-11-28T01:42:57,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/527bdceb055240d7b349e0758ecd8149, entries=200, sequenceid=243, filesize=14.2 K 2024-11-28T01:42:57,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/b6a7daf66c994330ac079199bfe1325f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/b6a7daf66c994330ac079199bfe1325f 2024-11-28T01:42:57,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/b6a7daf66c994330ac079199bfe1325f, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T01:42:57,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/d36ba121e2484d41b11043195616be68 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/d36ba121e2484d41b11043195616be68 2024-11-28T01:42:57,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/d36ba121e2484d41b11043195616be68, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T01:42:57,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9f37285715eee60df10bafa1eb0d60a0 in 869ms, sequenceid=243, compaction requested=true 2024-11-28T01:42:57,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:57,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:42:57,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:57,676 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:57,677 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:42:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:42:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:57,678 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:57,679 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:42:57,679 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,679 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/6450e77f82a4471ca25ee4216b81e5ff, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/04b527a22b734a5fbab0f93701660507, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/527bdceb055240d7b349e0758ecd8149] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=38.3 K 2024-11-28T01:42:57,679 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:57,679 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6450e77f82a4471ca25ee4216b81e5ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732758174387 2024-11-28T01:42:57,679 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:42:57,679 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,679 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/2834f6a4cf424228a2344b69ed46e2a9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8c9e30c2013a47bd8e8e5eda4f0aa01c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/b6a7daf66c994330ac079199bfe1325f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=35.9 K 2024-11-28T01:42:57,679 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2834f6a4cf424228a2344b69ed46e2a9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732758174387 2024-11-28T01:42:57,679 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b527a22b734a5fbab0f93701660507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732758174601 2024-11-28T01:42:57,680 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9e30c2013a47bd8e8e5eda4f0aa01c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732758174601 2024-11-28T01:42:57,680 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 527bdceb055240d7b349e0758ecd8149, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732758175748 2024-11-28T01:42:57,680 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b6a7daf66c994330ac079199bfe1325f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732758175748 2024-11-28T01:42:57,687 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:57,688 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/77447392756c4cb8a88bc834ff84b7e9 is 50, key is test_row_0/B:col10/1732758175762/Put/seqid=0 2024-11-28T01:42:57,690 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:57,691 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/3b89071ffb7e4bfaaef12f0fc10d8fb0 is 50, key is test_row_0/A:col10/1732758175762/Put/seqid=0 2024-11-28T01:42:57,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742390_1566 (size=12595) 2024-11-28T01:42:57,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742391_1567 (size=12595) 2024-11-28T01:42:57,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T01:42:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:57,816 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:42:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:57,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:57,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/5047e2fcf12a4a7e9f9f8984c028bf59 is 50, key is test_row_0/A:col10/1732758176847/Put/seqid=0 2024-11-28T01:42:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742392_1568 (size=12251) 2024-11-28T01:42:57,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:58,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758237991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758237994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T01:42:58,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758238082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758238082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758238083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,101 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/77447392756c4cb8a88bc834ff84b7e9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/77447392756c4cb8a88bc834ff84b7e9 2024-11-28T01:42:58,106 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into 77447392756c4cb8a88bc834ff84b7e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:58,106 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:58,106 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=13, startTime=1732758177676; duration=0sec 2024-11-28T01:42:58,106 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:42:58,106 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:42:58,106 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:42:58,106 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:42:58,107 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:42:58,107 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:58,107 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/79e5aece83b84c7ebc9d201f08f30356, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/4431bbcffa634cd6950117b0195df9d9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/d36ba121e2484d41b11043195616be68] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=35.9 K 2024-11-28T01:42:58,107 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e5aece83b84c7ebc9d201f08f30356, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732758174387 2024-11-28T01:42:58,107 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/3b89071ffb7e4bfaaef12f0fc10d8fb0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/3b89071ffb7e4bfaaef12f0fc10d8fb0 2024-11-28T01:42:58,107 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4431bbcffa634cd6950117b0195df9d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732758174601 2024-11-28T01:42:58,107 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d36ba121e2484d41b11043195616be68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732758175748 2024-11-28T01:42:58,111 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into 3b89071ffb7e4bfaaef12f0fc10d8fb0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:58,111 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:58,111 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=13, startTime=1732758177676; duration=0sec 2024-11-28T01:42:58,111 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:58,111 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:42:58,116 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:42:58,116 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/20bc805c8f584d3d881da28dba20c27f is 50, key is test_row_0/C:col10/1732758175762/Put/seqid=0 2024-11-28T01:42:58,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758238112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742393_1569 (size=12595) 2024-11-28T01:42:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758238113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/5047e2fcf12a4a7e9f9f8984c028bf59 2024-11-28T01:42:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/c3b6eb766a0141d69ab836a8c3773fd0 is 50, key is test_row_0/B:col10/1732758176847/Put/seqid=0 2024-11-28T01:42:58,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742394_1570 (size=12251) 2024-11-28T01:42:58,237 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/c3b6eb766a0141d69ab836a8c3773fd0 2024-11-28T01:42:58,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/bf2c7bedacfb4e85a454c0bd71716e3a is 50, key is test_row_0/C:col10/1732758176847/Put/seqid=0 2024-11-28T01:42:58,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742395_1571 (size=12251) 2024-11-28T01:42:58,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758238323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758238332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,530 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/20bc805c8f584d3d881da28dba20c27f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/20bc805c8f584d3d881da28dba20c27f 2024-11-28T01:42:58,534 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into 20bc805c8f584d3d881da28dba20c27f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:42:58,534 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:58,534 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=13, startTime=1732758177677; duration=0sec 2024-11-28T01:42:58,534 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:42:58,534 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:42:58,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758238642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,649 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/bf2c7bedacfb4e85a454c0bd71716e3a 2024-11-28T01:42:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/5047e2fcf12a4a7e9f9f8984c028bf59 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/5047e2fcf12a4a7e9f9f8984c028bf59 2024-11-28T01:42:58,657 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/5047e2fcf12a4a7e9f9f8984c028bf59, entries=150, sequenceid=263, filesize=12.0 K 2024-11-28T01:42:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/c3b6eb766a0141d69ab836a8c3773fd0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c3b6eb766a0141d69ab836a8c3773fd0 2024-11-28T01:42:58,660 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c3b6eb766a0141d69ab836a8c3773fd0, entries=150, sequenceid=263, filesize=12.0 K 2024-11-28T01:42:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/bf2c7bedacfb4e85a454c0bd71716e3a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/bf2c7bedacfb4e85a454c0bd71716e3a 2024-11-28T01:42:58,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758238653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:58,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:58,665 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/bf2c7bedacfb4e85a454c0bd71716e3a, entries=150, sequenceid=263, filesize=12.0 K 2024-11-28T01:42:58,666 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 9f37285715eee60df10bafa1eb0d60a0 in 850ms, sequenceid=263, compaction requested=false 2024-11-28T01:42:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:42:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-28T01:42:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-28T01:42:58,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-28T01:42:58,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7700 sec 2024-11-28T01:42:58,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7780 sec 2024-11-28T01:42:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T01:42:59,043 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:42:59,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:42:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-28T01:42:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T01:42:59,046 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:42:59,047 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:42:59,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:42:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:42:59,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:42:59,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:42:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:42:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:42:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:42:59,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/051bca60007d4874a4d492fca285e3cd is 50, key is test_row_0/A:col10/1732758179107/Put/seqid=0 2024-11-28T01:42:59,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742396_1572 (size=14741) 2024-11-28T01:42:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T01:42:59,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758239155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758239159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758239161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758239162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758239174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:42:59,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:59,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758239282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758239282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758239282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758239282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:42:59,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:59,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T01:42:59,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:42:59,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:59,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758239502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758239503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758239503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758239503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/051bca60007d4874a4d492fca285e3cd 2024-11-28T01:42:59,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/a2971dbac1484529bda4ce359947eb96 is 50, key is test_row_0/B:col10/1732758179107/Put/seqid=0 2024-11-28T01:42:59,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742397_1573 (size=12301) 2024-11-28T01:42:59,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:42:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T01:42:59,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:42:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758239812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758239812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758239812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758239813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:42:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:42:59,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/a2971dbac1484529bda4ce359947eb96 2024-11-28T01:42:59,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9ece6fd6943441c48505dce9f195eda0 is 50, key is test_row_0/C:col10/1732758179107/Put/seqid=0 2024-11-28T01:42:59,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742398_1574 (size=12301) 2024-11-28T01:42:59,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:42:59,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:42:59,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:42:59,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:42:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:00,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:43:00,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:00,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:00,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T01:43:00,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758240182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:00,191 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:00,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:43:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:00,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9ece6fd6943441c48505dce9f195eda0 2024-11-28T01:43:00,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758240333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758240333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758240333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758240333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:00,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/051bca60007d4874a4d492fca285e3cd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/051bca60007d4874a4d492fca285e3cd 2024-11-28T01:43:00,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/051bca60007d4874a4d492fca285e3cd, entries=200, sequenceid=279, filesize=14.4 K 2024-11-28T01:43:00,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/a2971dbac1484529bda4ce359947eb96 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a2971dbac1484529bda4ce359947eb96 2024-11-28T01:43:00,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a2971dbac1484529bda4ce359947eb96, entries=150, sequenceid=279, filesize=12.0 K 2024-11-28T01:43:00,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9ece6fd6943441c48505dce9f195eda0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9ece6fd6943441c48505dce9f195eda0 2024-11-28T01:43:00,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9ece6fd6943441c48505dce9f195eda0, entries=150, sequenceid=279, filesize=12.0 K 2024-11-28T01:43:00,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 9f37285715eee60df10bafa1eb0d60a0 in 1257ms, sequenceid=279, compaction requested=true 2024-11-28T01:43:00,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:00,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:00,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:00,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:00,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:00,365 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:00,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:00,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T01:43:00,366 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39587 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:00,366 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:43:00,367 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,367 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/3b89071ffb7e4bfaaef12f0fc10d8fb0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/5047e2fcf12a4a7e9f9f8984c028bf59, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/051bca60007d4874a4d492fca285e3cd] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=38.7 K 2024-11-28T01:43:00,367 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b89071ffb7e4bfaaef12f0fc10d8fb0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732758175748 2024-11-28T01:43:00,367 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5047e2fcf12a4a7e9f9f8984c028bf59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732758176847 2024-11-28T01:43:00,367 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 051bca60007d4874a4d492fca285e3cd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732758177983 2024-11-28T01:43:00,370 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:00,373 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37147 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:00,373 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:43:00,373 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,373 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/77447392756c4cb8a88bc834ff84b7e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c3b6eb766a0141d69ab836a8c3773fd0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a2971dbac1484529bda4ce359947eb96] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=36.3 K 2024-11-28T01:43:00,374 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 77447392756c4cb8a88bc834ff84b7e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732758175748 2024-11-28T01:43:00,374 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b6eb766a0141d69ab836a8c3773fd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732758176847 2024-11-28T01:43:00,374 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a2971dbac1484529bda4ce359947eb96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732758177989 2024-11-28T01:43:00,381 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:00,382 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/71ef7a95d6a64cba99a87cb72ca01b50 is 50, key is test_row_0/A:col10/1732758179107/Put/seqid=0 2024-11-28T01:43:00,382 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:00,382 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/a0bf76c7299440f5826a9c61ad59b089 is 50, key is test_row_0/B:col10/1732758179107/Put/seqid=0 2024-11-28T01:43:00,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742399_1575 (size=12847) 2024-11-28T01:43:00,393 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/71ef7a95d6a64cba99a87cb72ca01b50 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71ef7a95d6a64cba99a87cb72ca01b50 2024-11-28T01:43:00,397 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into 71ef7a95d6a64cba99a87cb72ca01b50(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:00,397 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:00,397 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=13, startTime=1732758180365; duration=0sec 2024-11-28T01:43:00,397 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:00,397 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:43:00,397 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:00,399 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37147 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:00,399 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:43:00,399 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,399 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/20bc805c8f584d3d881da28dba20c27f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/bf2c7bedacfb4e85a454c0bd71716e3a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9ece6fd6943441c48505dce9f195eda0] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=36.3 K 2024-11-28T01:43:00,400 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20bc805c8f584d3d881da28dba20c27f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732758175748 2024-11-28T01:43:00,402 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf2c7bedacfb4e85a454c0bd71716e3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732758176847 2024-11-28T01:43:00,403 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ece6fd6943441c48505dce9f195eda0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732758177989 2024-11-28T01:43:00,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742400_1576 (size=12847) 2024-11-28T01:43:00,414 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/a0bf76c7299440f5826a9c61ad59b089 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a0bf76c7299440f5826a9c61ad59b089 2024-11-28T01:43:00,420 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#489 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:00,420 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into a0bf76c7299440f5826a9c61ad59b089(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:00,420 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:00,420 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=13, startTime=1732758180365; duration=0sec 2024-11-28T01:43:00,421 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:00,421 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:43:00,421 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ec268f2867dc44198cfc421c94008544 is 50, key is test_row_0/C:col10/1732758179107/Put/seqid=0 2024-11-28T01:43:00,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T01:43:00,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:00,424 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T01:43:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:00,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/d63512a9dd1f4220b4d05285594a4f07 is 50, key is test_row_0/A:col10/1732758179159/Put/seqid=0 2024-11-28T01:43:00,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742401_1577 (size=12847) 2024-11-28T01:43:00,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742402_1578 (size=12301) 2024-11-28T01:43:00,843 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ec268f2867dc44198cfc421c94008544 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ec268f2867dc44198cfc421c94008544 2024-11-28T01:43:00,846 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/d63512a9dd1f4220b4d05285594a4f07 2024-11-28T01:43:00,847 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into ec268f2867dc44198cfc421c94008544(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:00,848 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:00,848 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=13, startTime=1732758180365; duration=0sec 2024-11-28T01:43:00,848 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:00,848 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:43:00,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/27005d0e11c840fdb6edae447f4768a9 is 50, key is test_row_0/B:col10/1732758179159/Put/seqid=0 2024-11-28T01:43:00,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742403_1579 (size=12301) 2024-11-28T01:43:00,859 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/27005d0e11c840fdb6edae447f4768a9 2024-11-28T01:43:00,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/0638bd85aa4e4d4fabcda81bc9238b3a is 50, key is test_row_0/C:col10/1732758179159/Put/seqid=0 2024-11-28T01:43:00,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742404_1580 (size=12301) 2024-11-28T01:43:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T01:43:01,276 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/0638bd85aa4e4d4fabcda81bc9238b3a 2024-11-28T01:43:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/d63512a9dd1f4220b4d05285594a4f07 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d63512a9dd1f4220b4d05285594a4f07 2024-11-28T01:43:01,284 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d63512a9dd1f4220b4d05285594a4f07, entries=150, sequenceid=300, filesize=12.0 K 2024-11-28T01:43:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/27005d0e11c840fdb6edae447f4768a9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/27005d0e11c840fdb6edae447f4768a9 2024-11-28T01:43:01,288 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/27005d0e11c840fdb6edae447f4768a9, entries=150, sequenceid=300, filesize=12.0 K 2024-11-28T01:43:01,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/0638bd85aa4e4d4fabcda81bc9238b3a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/0638bd85aa4e4d4fabcda81bc9238b3a 2024-11-28T01:43:01,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/0638bd85aa4e4d4fabcda81bc9238b3a, entries=150, sequenceid=300, filesize=12.0 K 2024-11-28T01:43:01,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 9f37285715eee60df10bafa1eb0d60a0 in 875ms, sequenceid=300, compaction requested=false 2024-11-28T01:43:01,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:01,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:01,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-28T01:43:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-28T01:43:01,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-28T01:43:01,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2530 sec 2024-11-28T01:43:01,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.2590 sec 2024-11-28T01:43:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:01,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:43:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:01,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/9df0c28574ee48f5a260d2a3709e65d4 is 50, key is test_row_0/A:col10/1732758181369/Put/seqid=0 2024-11-28T01:43:01,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742405_1581 (size=14741) 2024-11-28T01:43:01,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758241417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758241419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758241424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758241425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758241542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758241542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758241542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758241553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758241763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758241763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758241763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758241772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:01,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/9df0c28574ee48f5a260d2a3709e65d4 2024-11-28T01:43:01,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/20565542fa8344d6be03801e0fc3cf93 is 50, key is test_row_0/B:col10/1732758181369/Put/seqid=0 2024-11-28T01:43:01,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742406_1582 (size=12301) 2024-11-28T01:43:01,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/20565542fa8344d6be03801e0fc3cf93 2024-11-28T01:43:01,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ea8ac89dcb9243e9b26fb75a5da6fcb1 is 50, key is test_row_0/C:col10/1732758181369/Put/seqid=0 2024-11-28T01:43:01,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742407_1583 (size=12301) 2024-11-28T01:43:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758242083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758242083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758242083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758242083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ea8ac89dcb9243e9b26fb75a5da6fcb1 2024-11-28T01:43:02,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/9df0c28574ee48f5a260d2a3709e65d4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/9df0c28574ee48f5a260d2a3709e65d4 2024-11-28T01:43:02,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/9df0c28574ee48f5a260d2a3709e65d4, entries=200, sequenceid=313, filesize=14.4 K 2024-11-28T01:43:02,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/20565542fa8344d6be03801e0fc3cf93 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/20565542fa8344d6be03801e0fc3cf93 2024-11-28T01:43:02,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/20565542fa8344d6be03801e0fc3cf93, entries=150, sequenceid=313, filesize=12.0 K 2024-11-28T01:43:02,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ea8ac89dcb9243e9b26fb75a5da6fcb1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ea8ac89dcb9243e9b26fb75a5da6fcb1 2024-11-28T01:43:02,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ea8ac89dcb9243e9b26fb75a5da6fcb1, entries=150, sequenceid=313, filesize=12.0 K 2024-11-28T01:43:02,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 9f37285715eee60df10bafa1eb0d60a0 in 848ms, sequenceid=313, compaction requested=true 2024-11-28T01:43:02,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:02,219 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:02,219 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:02,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:02,220 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:02,220 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:02,220 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:43:02,220 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:43:02,220 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:02,220 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71ef7a95d6a64cba99a87cb72ca01b50, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d63512a9dd1f4220b4d05285594a4f07, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/9df0c28574ee48f5a260d2a3709e65d4] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=39.0 K 2024-11-28T01:43:02,220 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:02,220 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a0bf76c7299440f5826a9c61ad59b089, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/27005d0e11c840fdb6edae447f4768a9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/20565542fa8344d6be03801e0fc3cf93] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=36.6 K 2024-11-28T01:43:02,220 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71ef7a95d6a64cba99a87cb72ca01b50, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732758177989 2024-11-28T01:43:02,220 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a0bf76c7299440f5826a9c61ad59b089, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732758177989 2024-11-28T01:43:02,221 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d63512a9dd1f4220b4d05285594a4f07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732758179146 2024-11-28T01:43:02,221 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 27005d0e11c840fdb6edae447f4768a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732758179146 2024-11-28T01:43:02,221 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9df0c28574ee48f5a260d2a3709e65d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732758181369 2024-11-28T01:43:02,221 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 20565542fa8344d6be03801e0fc3cf93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732758181369 2024-11-28T01:43:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:02,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T01:43:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:02,229 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:02,229 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/1b918f27ffdb4b9abfa0c8f23c2caada is 50, key is test_row_0/B:col10/1732758181369/Put/seqid=0 2024-11-28T01:43:02,231 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:02,232 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/2360cd94d0a24a4a9adcfdc3802943f4 is 50, key is test_row_0/A:col10/1732758181369/Put/seqid=0 2024-11-28T01:43:02,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/0c6e9c6879104569ad516d4a3d71a10e is 50, key is test_row_0/A:col10/1732758181399/Put/seqid=0 2024-11-28T01:43:02,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742409_1585 (size=12949) 2024-11-28T01:43:02,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742408_1584 (size=12949) 2024-11-28T01:43:02,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742410_1586 (size=14741) 2024-11-28T01:43:02,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/0c6e9c6879104569ad516d4a3d71a10e 2024-11-28T01:43:02,254 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/1b918f27ffdb4b9abfa0c8f23c2caada as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1b918f27ffdb4b9abfa0c8f23c2caada 2024-11-28T01:43:02,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d687b7ea932a47c9bde0346b55a1e175 is 50, key is test_row_0/B:col10/1732758181399/Put/seqid=0 2024-11-28T01:43:02,263 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into 1b918f27ffdb4b9abfa0c8f23c2caada(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:02,263 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:02,263 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=13, startTime=1732758182219; duration=0sec 2024-11-28T01:43:02,263 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:02,263 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:43:02,263 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:02,264 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:02,264 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:43:02,264 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:02,264 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ec268f2867dc44198cfc421c94008544, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/0638bd85aa4e4d4fabcda81bc9238b3a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ea8ac89dcb9243e9b26fb75a5da6fcb1] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=36.6 K 2024-11-28T01:43:02,265 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ec268f2867dc44198cfc421c94008544, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732758177989 2024-11-28T01:43:02,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742411_1587 (size=12301) 2024-11-28T01:43:02,266 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0638bd85aa4e4d4fabcda81bc9238b3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732758179146 2024-11-28T01:43:02,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d687b7ea932a47c9bde0346b55a1e175 2024-11-28T01:43:02,266 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8ac89dcb9243e9b26fb75a5da6fcb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732758181369 2024-11-28T01:43:02,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/e937c19e26eb48f091a9f1f103513476 is 50, key is test_row_0/C:col10/1732758181399/Put/seqid=0 2024-11-28T01:43:02,275 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:02,276 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/76884d7866df4e68b1cdda1d3d8a789e is 50, key is test_row_0/C:col10/1732758181369/Put/seqid=0 2024-11-28T01:43:02,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742412_1588 (size=12301) 2024-11-28T01:43:02,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742413_1589 (size=12949) 2024-11-28T01:43:02,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/e937c19e26eb48f091a9f1f103513476 2024-11-28T01:43:02,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/0c6e9c6879104569ad516d4a3d71a10e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0c6e9c6879104569ad516d4a3d71a10e 2024-11-28T01:43:02,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0c6e9c6879104569ad516d4a3d71a10e, entries=200, sequenceid=334, filesize=14.4 K 2024-11-28T01:43:02,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d687b7ea932a47c9bde0346b55a1e175 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d687b7ea932a47c9bde0346b55a1e175 2024-11-28T01:43:02,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d687b7ea932a47c9bde0346b55a1e175, entries=150, sequenceid=334, filesize=12.0 K 2024-11-28T01:43:02,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/e937c19e26eb48f091a9f1f103513476 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e937c19e26eb48f091a9f1f103513476 2024-11-28T01:43:02,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e937c19e26eb48f091a9f1f103513476, entries=150, sequenceid=334, filesize=12.0 K 2024-11-28T01:43:02,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=40.25 KB/41220 for 9f37285715eee60df10bafa1eb0d60a0 in 73ms, sequenceid=334, compaction requested=false 2024-11-28T01:43:02,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:02,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:43:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:02,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:02,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/631eaaaa99dd42f69bdc892c4917ac1b is 50, key is test_row_0/A:col10/1732758182262/Put/seqid=0 2024-11-28T01:43:02,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742414_1590 (size=14741) 2024-11-28T01:43:02,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/631eaaaa99dd42f69bdc892c4917ac1b 2024-11-28T01:43:02,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/600a7770d38240bf8c3284095a7ac9e4 is 50, key is test_row_0/B:col10/1732758182262/Put/seqid=0 2024-11-28T01:43:02,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742415_1591 (size=12301) 2024-11-28T01:43:02,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758182470, max timestamp=1732758182497Num uniques 859; 2024-11-28T01:43:02,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:43:02,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758242563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758242602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758242603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758242603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758242603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,645 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/2360cd94d0a24a4a9adcfdc3802943f4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/2360cd94d0a24a4a9adcfdc3802943f4 2024-11-28T01:43:02,653 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into 2360cd94d0a24a4a9adcfdc3802943f4(size=12.6 K), total size for store is 27.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:02,653 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:02,653 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=13, startTime=1732758182218; duration=0sec 2024-11-28T01:43:02,654 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:02,654 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:43:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758242672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:02,686 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/76884d7866df4e68b1cdda1d3d8a789e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/76884d7866df4e68b1cdda1d3d8a789e 2024-11-28T01:43:02,690 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into 76884d7866df4e68b1cdda1d3d8a789e(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:02,690 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:02,690 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=13, startTime=1732758182219; duration=0sec 2024-11-28T01:43:02,690 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:02,690 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:43:02,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/600a7770d38240bf8c3284095a7ac9e4 2024-11-28T01:43:02,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/927e1768584644fd89728f0e91a01770 is 50, key is test_row_0/C:col10/1732758182262/Put/seqid=0 2024-11-28T01:43:02,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742416_1592 (size=12301) 2024-11-28T01:43:02,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758242892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:02,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:02,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/927e1768584644fd89728f0e91a01770 2024-11-28T01:43:03,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/631eaaaa99dd42f69bdc892c4917ac1b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/631eaaaa99dd42f69bdc892c4917ac1b 2024-11-28T01:43:03,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/631eaaaa99dd42f69bdc892c4917ac1b, entries=200, sequenceid=345, filesize=14.4 K 2024-11-28T01:43:03,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/600a7770d38240bf8c3284095a7ac9e4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/600a7770d38240bf8c3284095a7ac9e4 2024-11-28T01:43:03,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/600a7770d38240bf8c3284095a7ac9e4, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T01:43:03,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/927e1768584644fd89728f0e91a01770 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/927e1768584644fd89728f0e91a01770 2024-11-28T01:43:03,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/927e1768584644fd89728f0e91a01770, entries=150, sequenceid=345, filesize=12.0 K 2024-11-28T01:43:03,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for 9f37285715eee60df10bafa1eb0d60a0 in 859ms, sequenceid=345, compaction requested=true 2024-11-28T01:43:03,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:03,162 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:03,163 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:03,163 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:03,163 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:43:03,163 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,163 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/2360cd94d0a24a4a9adcfdc3802943f4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0c6e9c6879104569ad516d4a3d71a10e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/631eaaaa99dd42f69bdc892c4917ac1b] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=41.4 K 2024-11-28T01:43:03,164 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2360cd94d0a24a4a9adcfdc3802943f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732758181369 2024-11-28T01:43:03,164 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c6e9c6879104569ad516d4a3d71a10e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732758181399 2024-11-28T01:43:03,164 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:03,164 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:43:03,164 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,164 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1b918f27ffdb4b9abfa0c8f23c2caada, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d687b7ea932a47c9bde0346b55a1e175, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/600a7770d38240bf8c3284095a7ac9e4] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=36.7 K 2024-11-28T01:43:03,165 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631eaaaa99dd42f69bdc892c4917ac1b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758182235 2024-11-28T01:43:03,165 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b918f27ffdb4b9abfa0c8f23c2caada, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732758181369 2024-11-28T01:43:03,165 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d687b7ea932a47c9bde0346b55a1e175, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732758181399 2024-11-28T01:43:03,165 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 600a7770d38240bf8c3284095a7ac9e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758182235 2024-11-28T01:43:03,174 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:03,174 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:03,175 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/25007ff03d0747cb9b3f5cb857995baf is 50, key is test_row_0/B:col10/1732758182262/Put/seqid=0 2024-11-28T01:43:03,175 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/10aa8d5075f1432f9e3d066c03b3a180 is 50, key is test_row_0/A:col10/1732758182262/Put/seqid=0 2024-11-28T01:43:03,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742417_1593 (size=13051) 2024-11-28T01:43:03,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742418_1594 (size=13051) 2024-11-28T01:43:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T01:43:03,203 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:03,203 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-28T01:43:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T01:43:03,205 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:03,206 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:03,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:03,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=136.42 KB heapSize=401.56 KB 2024-11-28T01:43:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:03,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/799c618572984230aa3a96843617afe4 is 50, key is test_row_0/A:col10/1732758183212/Put/seqid=0 2024-11-28T01:43:03,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742419_1595 (size=14741) 2024-11-28T01:43:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758243273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T01:43:03,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T01:43:03,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:03,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758243392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T01:43:03,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:03,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T01:43:03,589 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/10aa8d5075f1432f9e3d066c03b3a180 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/10aa8d5075f1432f9e3d066c03b3a180 2024-11-28T01:43:03,593 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into 10aa8d5075f1432f9e3d066c03b3a180(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:03,593 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:03,594 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=13, startTime=1732758183162; duration=0sec 2024-11-28T01:43:03,594 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:03,594 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:43:03,594 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:03,594 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:03,595 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:43:03,595 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,595 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/76884d7866df4e68b1cdda1d3d8a789e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e937c19e26eb48f091a9f1f103513476, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/927e1768584644fd89728f0e91a01770] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=36.7 K 2024-11-28T01:43:03,595 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76884d7866df4e68b1cdda1d3d8a789e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732758181369 2024-11-28T01:43:03,595 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e937c19e26eb48f091a9f1f103513476, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732758181399 2024-11-28T01:43:03,596 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 927e1768584644fd89728f0e91a01770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758182235 2024-11-28T01:43:03,596 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/25007ff03d0747cb9b3f5cb857995baf as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/25007ff03d0747cb9b3f5cb857995baf 2024-11-28T01:43:03,601 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into 25007ff03d0747cb9b3f5cb857995baf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:03,601 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:03,601 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=13, startTime=1732758183163; duration=0sec 2024-11-28T01:43:03,601 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:03,601 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:43:03,604 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:03,604 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/020d5ecd4d6e4a1b8ee5e3e3ffd56598 is 50, key is test_row_0/C:col10/1732758182262/Put/seqid=0 2024-11-28T01:43:03,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742420_1596 (size=13051) 2024-11-28T01:43:03,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758243612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/799c618572984230aa3a96843617afe4 2024-11-28T01:43:03,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/aa720cf184bb446bbd146bff7484bcc8 is 50, key is test_row_0/B:col10/1732758183212/Put/seqid=0 2024-11-28T01:43:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758243623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758243623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758243623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758243623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,636 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,637 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,637 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,638 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:03,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742421_1597 (size=12301) 2024-11-28T01:43:03,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T01:43:03,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:03,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T01:43:03,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:03,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T01:43:03,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758243932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:03,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T01:43:03,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:03,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:03,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:04,012 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/020d5ecd4d6e4a1b8ee5e3e3ffd56598 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020d5ecd4d6e4a1b8ee5e3e3ffd56598 2024-11-28T01:43:04,016 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into 020d5ecd4d6e4a1b8ee5e3e3ffd56598(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:04,016 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:04,016 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=13, startTime=1732758183163; duration=0sec 2024-11-28T01:43:04,016 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:04,016 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:43:04,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/aa720cf184bb446bbd146bff7484bcc8 2024-11-28T01:43:04,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/f7e3041091dd4c8b9ca2403f94f1950b is 50, key is test_row_0/C:col10/1732758183212/Put/seqid=0 2024-11-28T01:43:04,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742422_1598 (size=12301) 2024-11-28T01:43:04,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/f7e3041091dd4c8b9ca2403f94f1950b 2024-11-28T01:43:04,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/799c618572984230aa3a96843617afe4 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/799c618572984230aa3a96843617afe4 2024-11-28T01:43:04,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/799c618572984230aa3a96843617afe4, entries=200, sequenceid=375, filesize=14.4 K 2024-11-28T01:43:04,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/aa720cf184bb446bbd146bff7484bcc8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/aa720cf184bb446bbd146bff7484bcc8 2024-11-28T01:43:04,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/aa720cf184bb446bbd146bff7484bcc8, entries=150, sequenceid=375, filesize=12.0 K 2024-11-28T01:43:04,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/f7e3041091dd4c8b9ca2403f94f1950b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f7e3041091dd4c8b9ca2403f94f1950b 2024-11-28T01:43:04,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f7e3041091dd4c8b9ca2403f94f1950b, entries=150, sequenceid=375, filesize=12.0 K 2024-11-28T01:43:04,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~401.52 KB/411152, currentSize=40.25 KB/41220 for 9f37285715eee60df10bafa1eb0d60a0 in 861ms, sequenceid=375, compaction requested=false 2024-11-28T01:43:04,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:04,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T01:43:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:04,124 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-28T01:43:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:04,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:04,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/b9a5809ac57442b4918ea7715e62d50e is 50, key is test_row_0/A:col10/1732758183271/Put/seqid=0 2024-11-28T01:43:04,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742423_1599 (size=12301) 2024-11-28T01:43:04,137 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/b9a5809ac57442b4918ea7715e62d50e 2024-11-28T01:43:04,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d24511162c6140e7a14f22d4480ed8da is 50, key is test_row_0/B:col10/1732758183271/Put/seqid=0 2024-11-28T01:43:04,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742424_1600 (size=12301) 2024-11-28T01:43:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T01:43:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:04,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:04,549 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d24511162c6140e7a14f22d4480ed8da 2024-11-28T01:43:04,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/3d9f9c091e8d4663b83d045a792a514c is 50, key is test_row_0/C:col10/1732758183271/Put/seqid=0 2024-11-28T01:43:04,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742425_1601 (size=12301) 2024-11-28T01:43:04,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1732758184546, max timestamp=1732758184608Num uniques 859; 2024-11-28T01:43:04,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-28T01:43:04,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1732758184546, max timestamp=1732758184608Num uniques 859; 2024-11-28T01:43:04,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1732758184546, max timestamp=1732758184608Num uniques 859; 2024-11-28T01:43:04,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-28T01:43:04,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-11-28T01:43:04,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758184745, max timestamp=1732758184796Num uniques 968; 2024-11-28T01:43:04,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758184745, max timestamp=1732758184796Num uniques 968; 2024-11-28T01:43:04,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732758184745, max timestamp=1732758184796Num uniques 968; 2024-11-28T01:43:04,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-11-28T01:43:04,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-11-28T01:43:04,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-11-28T01:43:04,959 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/3d9f9c091e8d4663b83d045a792a514c 2024-11-28T01:43:04,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/b9a5809ac57442b4918ea7715e62d50e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b9a5809ac57442b4918ea7715e62d50e 2024-11-28T01:43:04,966 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b9a5809ac57442b4918ea7715e62d50e, entries=150, sequenceid=387, filesize=12.0 K 2024-11-28T01:43:04,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d24511162c6140e7a14f22d4480ed8da as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d24511162c6140e7a14f22d4480ed8da 2024-11-28T01:43:04,971 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d24511162c6140e7a14f22d4480ed8da, entries=150, sequenceid=387, filesize=12.0 K 2024-11-28T01:43:04,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/3d9f9c091e8d4663b83d045a792a514c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/3d9f9c091e8d4663b83d045a792a514c 2024-11-28T01:43:04,975 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/3d9f9c091e8d4663b83d045a792a514c, entries=150, sequenceid=387, filesize=12.0 K 2024-11-28T01:43:04,975 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=127.47 KB/130530 for 9f37285715eee60df10bafa1eb0d60a0 in 851ms, sequenceid=387, compaction requested=true 2024-11-28T01:43:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-28T01:43:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-28T01:43:04,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-28T01:43:04,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-11-28T01:43:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:04,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=134.18 KB heapSize=384.14 KB 2024-11-28T01:43:04,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.7760 sec 2024-11-28T01:43:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:04,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/1249499801604d01ba31fc3a2cd38d57 is 50, key is test_row_0/A:col10/1732758184944/Put/seqid=0 2024-11-28T01:43:04,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742426_1602 (size=14741) 2024-11-28T01:43:04,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/1249499801604d01ba31fc3a2cd38d57 2024-11-28T01:43:04,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d9a0f928c4d74d2b82b38b114dba3fe7 is 50, key is test_row_0/B:col10/1732758184944/Put/seqid=0 2024-11-28T01:43:05,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742427_1603 (size=12301) 2024-11-28T01:43:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758245061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758245172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T01:43:05,353 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:05,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-28T01:43:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T01:43:05,355 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:05,356 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:05,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:05,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d9a0f928c4d74d2b82b38b114dba3fe7 2024-11-28T01:43:05,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758245393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/1c92c1a11c554abe989a478a57fe589e is 50, key is test_row_0/C:col10/1732758184944/Put/seqid=0 2024-11-28T01:43:05,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742428_1604 (size=12301) 2024-11-28T01:43:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T01:43:05,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T01:43:05,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:05,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T01:43:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35100 deadline: 1732758245652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:05,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,662 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35132 deadline: 1732758245663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35146 deadline: 1732758245663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35110 deadline: 1732758245663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,666 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,667 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,668 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_1' in 9f37285715eee60df10bafa1eb0d60a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T01:43:05,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35070 deadline: 1732758245712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 , the old value is region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:05,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., hostname=9df3f25f5867,42371,1732758050017, seqNum=2 because the exception is null or not the one we care about 2024-11-28T01:43:05,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/1c92c1a11c554abe989a478a57fe589e 2024-11-28T01:43:05,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T01:43:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:05,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/1249499801604d01ba31fc3a2cd38d57 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/1249499801604d01ba31fc3a2cd38d57 2024-11-28T01:43:05,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/1249499801604d01ba31fc3a2cd38d57, entries=200, sequenceid=441, filesize=14.4 K 2024-11-28T01:43:05,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/d9a0f928c4d74d2b82b38b114dba3fe7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d9a0f928c4d74d2b82b38b114dba3fe7 2024-11-28T01:43:05,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d9a0f928c4d74d2b82b38b114dba3fe7, entries=150, sequenceid=441, filesize=12.0 K 2024-11-28T01:43:05,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/1c92c1a11c554abe989a478a57fe589e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/1c92c1a11c554abe989a478a57fe589e 2024-11-28T01:43:05,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/1c92c1a11c554abe989a478a57fe589e, entries=150, sequenceid=441, filesize=12.0 K 2024-11-28T01:43:05,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.09 KB/393312, currentSize=46.96 KB/48090 for 9f37285715eee60df10bafa1eb0d60a0 in 850ms, sequenceid=441, compaction requested=true 2024-11-28T01:43:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:05,831 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:43:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:05,831 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:43:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9f37285715eee60df10bafa1eb0d60a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:05,832 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:43:05,832 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/A is initiating minor compaction (all files) 2024-11-28T01:43:05,832 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/A in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,832 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/10aa8d5075f1432f9e3d066c03b3a180, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/799c618572984230aa3a96843617afe4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b9a5809ac57442b4918ea7715e62d50e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/1249499801604d01ba31fc3a2cd38d57] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=53.5 K 2024-11-28T01:43:05,833 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:43:05,833 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/B is initiating minor compaction (all files) 2024-11-28T01:43:05,833 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/B in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,833 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/25007ff03d0747cb9b3f5cb857995baf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/aa720cf184bb446bbd146bff7484bcc8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d24511162c6140e7a14f22d4480ed8da, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d9a0f928c4d74d2b82b38b114dba3fe7] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=48.8 K 2024-11-28T01:43:05,833 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10aa8d5075f1432f9e3d066c03b3a180, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758182235 2024-11-28T01:43:05,833 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 25007ff03d0747cb9b3f5cb857995baf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758182235 2024-11-28T01:43:05,833 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 799c618572984230aa3a96843617afe4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732758182497 2024-11-28T01:43:05,834 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting aa720cf184bb446bbd146bff7484bcc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732758182497 2024-11-28T01:43:05,834 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9a5809ac57442b4918ea7715e62d50e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732758183242 2024-11-28T01:43:05,834 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d24511162c6140e7a14f22d4480ed8da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732758183242 2024-11-28T01:43:05,834 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a0f928c4d74d2b82b38b114dba3fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732758184944 2024-11-28T01:43:05,834 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1249499801604d01ba31fc3a2cd38d57, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732758184884 2024-11-28T01:43:05,844 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#A#compaction#517 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:05,844 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/a20a3d995c25403ba5bb61a38e0779ef is 50, key is test_row_0/A:col10/1732758184944/Put/seqid=0 2024-11-28T01:43:05,847 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#B#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:05,848 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/7ac20511bf974bf998f17b45ae5ca63f is 50, key is test_row_0/B:col10/1732758184944/Put/seqid=0 2024-11-28T01:43:05,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742429_1605 (size=13187) 2024-11-28T01:43:05,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742430_1606 (size=13187) 2024-11-28T01:43:05,878 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/7ac20511bf974bf998f17b45ae5ca63f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/7ac20511bf974bf998f17b45ae5ca63f 2024-11-28T01:43:05,882 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/B of 9f37285715eee60df10bafa1eb0d60a0 into 7ac20511bf974bf998f17b45ae5ca63f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:05,882 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:05,882 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/B, priority=12, startTime=1732758185831; duration=0sec 2024-11-28T01:43:05,883 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:05,883 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:B 2024-11-28T01:43:05,883 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:43:05,885 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:43:05,885 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): 9f37285715eee60df10bafa1eb0d60a0/C is initiating minor compaction (all files) 2024-11-28T01:43:05,885 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9f37285715eee60df10bafa1eb0d60a0/C in TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,885 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020d5ecd4d6e4a1b8ee5e3e3ffd56598, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f7e3041091dd4c8b9ca2403f94f1950b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/3d9f9c091e8d4663b83d045a792a514c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/1c92c1a11c554abe989a478a57fe589e] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp, totalSize=48.8 K 2024-11-28T01:43:05,885 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 020d5ecd4d6e4a1b8ee5e3e3ffd56598, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732758182235 2024-11-28T01:43:05,886 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e3041091dd4c8b9ca2403f94f1950b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732758182497 2024-11-28T01:43:05,886 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9f9c091e8d4663b83d045a792a514c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732758183242 2024-11-28T01:43:05,886 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c92c1a11c554abe989a478a57fe589e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732758184944 2024-11-28T01:43:05,895 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9f37285715eee60df10bafa1eb0d60a0#C#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:05,896 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/2f8ab65caa1443a0a9b07f3772f8c874 is 50, key is test_row_0/C:col10/1732758184944/Put/seqid=0 2024-11-28T01:43:05,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742431_1607 (size=13187) 2024-11-28T01:43:05,911 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/2f8ab65caa1443a0a9b07f3772f8c874 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2f8ab65caa1443a0a9b07f3772f8c874 2024-11-28T01:43:05,920 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/C of 9f37285715eee60df10bafa1eb0d60a0 into 2f8ab65caa1443a0a9b07f3772f8c874(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:05,920 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:05,920 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/C, priority=12, startTime=1732758185831; duration=0sec 2024-11-28T01:43:05,920 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:05,920 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:C 2024-11-28T01:43:05,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-28T01:43:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:05,968 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:43:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:05,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/4ef76cdaeced4095997b4114001c1b1d is 50, key is test_row_0/A:col10/1732758185032/Put/seqid=0 2024-11-28T01:43:05,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742432_1608 (size=12301) 2024-11-28T01:43:05,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/4ef76cdaeced4095997b4114001c1b1d 2024-11-28T01:43:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T01:43:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/67b901fb40fc45f49bae3938d1d46af1 is 50, key is test_row_0/B:col10/1732758185032/Put/seqid=0 2024-11-28T01:43:05,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742433_1609 (size=12301) 2024-11-28T01:43:06,013 INFO [Thread-2281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2281. 2024-11-28T01:43:06,013 DEBUG [Thread-2281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:06,013 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,013 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:06,015 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-28T01:43:06,015 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:06,015 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,015 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:06,016 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-11-28T01:43:06,016 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:06,016 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,016 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:06,017 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-28T01:43:06,018 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:06,018 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,018 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:06,018 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-28T01:43:06,018 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:06,018 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,018 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:06,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. as already flushing 2024-11-28T01:43:06,233 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-11-28T01:43:06,234 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:06,234 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,234 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:06,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:06,257 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/a20a3d995c25403ba5bb61a38e0779ef as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a20a3d995c25403ba5bb61a38e0779ef 2024-11-28T01:43:06,261 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9f37285715eee60df10bafa1eb0d60a0/A of 9f37285715eee60df10bafa1eb0d60a0 into a20a3d995c25403ba5bb61a38e0779ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:06,261 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:06,261 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., storeName=9f37285715eee60df10bafa1eb0d60a0/A, priority=12, startTime=1732758185831; duration=0sec 2024-11-28T01:43:06,261 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:06,261 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9f37285715eee60df10bafa1eb0d60a0:A 2024-11-28T01:43:06,395 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/67b901fb40fc45f49bae3938d1d46af1 2024-11-28T01:43:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9b836a3fcb6045899f1d5f8af724a819 is 50, key is test_row_0/C:col10/1732758185032/Put/seqid=0 2024-11-28T01:43:06,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742434_1610 (size=12301) 2024-11-28T01:43:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T01:43:06,804 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9b836a3fcb6045899f1d5f8af724a819 2024-11-28T01:43:06,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/4ef76cdaeced4095997b4114001c1b1d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/4ef76cdaeced4095997b4114001c1b1d 2024-11-28T01:43:06,810 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/4ef76cdaeced4095997b4114001c1b1d, entries=150, sequenceid=453, filesize=12.0 K 2024-11-28T01:43:06,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/67b901fb40fc45f49bae3938d1d46af1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/67b901fb40fc45f49bae3938d1d46af1 2024-11-28T01:43:06,814 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/67b901fb40fc45f49bae3938d1d46af1, entries=150, sequenceid=453, filesize=12.0 K 2024-11-28T01:43:06,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/9b836a3fcb6045899f1d5f8af724a819 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9b836a3fcb6045899f1d5f8af724a819 2024-11-28T01:43:06,817 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9b836a3fcb6045899f1d5f8af724a819, entries=150, sequenceid=453, filesize=12.0 K 2024-11-28T01:43:06,817 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for 9f37285715eee60df10bafa1eb0d60a0 in 849ms, sequenceid=453, compaction requested=false 2024-11-28T01:43:06,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 9f37285715eee60df10bafa1eb0d60a0: 2024-11-28T01:43:06,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:06,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-28T01:43:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-28T01:43:06,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-28T01:43:06,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-11-28T01:43:06,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4670 sec 2024-11-28T01:43:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T01:43:07,503 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:09,684 INFO [Thread-2270 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2270. 2024-11-28T01:43:09,684 DEBUG [Thread-2270 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:09,684 INFO [Thread-2272 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2272. 2024-11-28T01:43:09,684 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,684 DEBUG [Thread-2272 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:09,684 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,684 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,684 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:09,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:09,694 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-11-28T01:43:09,694 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:09,694 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,694 INFO [Thread-2274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2274. 2024-11-28T01:43:09,694 DEBUG [Thread-2274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:09,694 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,694 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:09,694 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T01:43:09,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1036 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3108 rows 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1033 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3099 rows 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1037 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3111 rows 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1040 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3118 rows 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1041 2024-11-28T01:43:09,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3122 rows 2024-11-28T01:43:09,695 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:43:09,695 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:43:09,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:09,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:09,695 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T01:43:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T01:43:09,698 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758189698"}]},"ts":"1732758189698"} 2024-11-28T01:43:09,699 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T01:43:09,699 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T01:43:09,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:43:09,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, UNASSIGN}] 2024-11-28T01:43:09,702 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, UNASSIGN 2024-11-28T01:43:09,702 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9f37285715eee60df10bafa1eb0d60a0, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:09,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, UNASSIGN because future has completed 2024-11-28T01:43:09,703 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:43:09,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:43:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T01:43:09,855 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:09,855 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 9f37285715eee60df10bafa1eb0d60a0, disabling compactions & flushes 2024-11-28T01:43:09,856 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. after waiting 0 ms 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:09,856 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 9f37285715eee60df10bafa1eb0d60a0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=A 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=B 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9f37285715eee60df10bafa1eb0d60a0, store=C 2024-11-28T01:43:09,856 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:09,860 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/0aa2afc203e84ec9ac793b073090e04d is 50, key is test_row_1/A:col10/1732758189692/Put/seqid=0 2024-11-28T01:43:09,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742435_1611 (size=9857) 2024-11-28T01:43:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T01:43:10,263 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/0aa2afc203e84ec9ac793b073090e04d 2024-11-28T01:43:10,269 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/1545655572a24749b6752845cfea7fcc is 50, key is test_row_1/B:col10/1732758189692/Put/seqid=0 2024-11-28T01:43:10,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742436_1612 (size=9857) 2024-11-28T01:43:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T01:43:10,673 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/1545655572a24749b6752845cfea7fcc 2024-11-28T01:43:10,679 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ee6d5111887f4c43af7464cf09864402 is 50, key is test_row_1/C:col10/1732758189692/Put/seqid=0 2024-11-28T01:43:10,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742437_1613 (size=9857) 2024-11-28T01:43:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T01:43:11,083 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ee6d5111887f4c43af7464cf09864402 2024-11-28T01:43:11,086 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/A/0aa2afc203e84ec9ac793b073090e04d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0aa2afc203e84ec9ac793b073090e04d 2024-11-28T01:43:11,089 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0aa2afc203e84ec9ac793b073090e04d, entries=100, sequenceid=462, filesize=9.6 K 2024-11-28T01:43:11,090 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/B/1545655572a24749b6752845cfea7fcc as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1545655572a24749b6752845cfea7fcc 2024-11-28T01:43:11,092 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1545655572a24749b6752845cfea7fcc, entries=100, sequenceid=462, filesize=9.6 K 2024-11-28T01:43:11,092 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/.tmp/C/ee6d5111887f4c43af7464cf09864402 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ee6d5111887f4c43af7464cf09864402 2024-11-28T01:43:11,095 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ee6d5111887f4c43af7464cf09864402, entries=100, sequenceid=462, filesize=9.6 K 2024-11-28T01:43:11,096 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9f37285715eee60df10bafa1eb0d60a0 in 1240ms, sequenceid=462, compaction requested=true 2024-11-28T01:43:11,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b840ed34a6284a3ba8f9908f6c452837, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d0d4cf4c90fa4deaaad30f853ff4ee47, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/fb9db6fe9eb24be08c10e9efaaac8e55, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7496f887553a4d6f9f4e74ac5c4c8fdf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7cd48122a2864d2fbbde5fef443e88e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/f360a0ff5cde4a5f9243342180f1affb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/78f28ab1a85b4d68b52cbff1f99b6619, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/095ee18a3c13472189e379f399cb0d43, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/8cd2de9144a24613b1d5711ad68f19e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/95f47208dc43489c907e91406c816212, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/6450e77f82a4471ca25ee4216b81e5ff, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/04b527a22b734a5fbab0f93701660507, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/527bdceb055240d7b349e0758ecd8149, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/3b89071ffb7e4bfaaef12f0fc10d8fb0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/5047e2fcf12a4a7e9f9f8984c028bf59, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/051bca60007d4874a4d492fca285e3cd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71ef7a95d6a64cba99a87cb72ca01b50, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d63512a9dd1f4220b4d05285594a4f07, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/9df0c28574ee48f5a260d2a3709e65d4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/2360cd94d0a24a4a9adcfdc3802943f4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0c6e9c6879104569ad516d4a3d71a10e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/631eaaaa99dd42f69bdc892c4917ac1b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/10aa8d5075f1432f9e3d066c03b3a180, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/799c618572984230aa3a96843617afe4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b9a5809ac57442b4918ea7715e62d50e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/1249499801604d01ba31fc3a2cd38d57] to archive 2024-11-28T01:43:11,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:43:11,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b840ed34a6284a3ba8f9908f6c452837 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b840ed34a6284a3ba8f9908f6c452837 2024-11-28T01:43:11,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d0d4cf4c90fa4deaaad30f853ff4ee47 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d0d4cf4c90fa4deaaad30f853ff4ee47 2024-11-28T01:43:11,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/fb9db6fe9eb24be08c10e9efaaac8e55 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/fb9db6fe9eb24be08c10e9efaaac8e55 2024-11-28T01:43:11,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7496f887553a4d6f9f4e74ac5c4c8fdf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7496f887553a4d6f9f4e74ac5c4c8fdf 2024-11-28T01:43:11,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7cd48122a2864d2fbbde5fef443e88e4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/7cd48122a2864d2fbbde5fef443e88e4 2024-11-28T01:43:11,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/f360a0ff5cde4a5f9243342180f1affb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/f360a0ff5cde4a5f9243342180f1affb 2024-11-28T01:43:11,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/78f28ab1a85b4d68b52cbff1f99b6619 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/78f28ab1a85b4d68b52cbff1f99b6619 2024-11-28T01:43:11,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/095ee18a3c13472189e379f399cb0d43 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/095ee18a3c13472189e379f399cb0d43 2024-11-28T01:43:11,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/8cd2de9144a24613b1d5711ad68f19e3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/8cd2de9144a24613b1d5711ad68f19e3 2024-11-28T01:43:11,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/95f47208dc43489c907e91406c816212 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/95f47208dc43489c907e91406c816212 2024-11-28T01:43:11,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/6450e77f82a4471ca25ee4216b81e5ff to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/6450e77f82a4471ca25ee4216b81e5ff 2024-11-28T01:43:11,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/04b527a22b734a5fbab0f93701660507 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/04b527a22b734a5fbab0f93701660507 2024-11-28T01:43:11,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/527bdceb055240d7b349e0758ecd8149 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/527bdceb055240d7b349e0758ecd8149 2024-11-28T01:43:11,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/3b89071ffb7e4bfaaef12f0fc10d8fb0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/3b89071ffb7e4bfaaef12f0fc10d8fb0 2024-11-28T01:43:11,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/5047e2fcf12a4a7e9f9f8984c028bf59 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/5047e2fcf12a4a7e9f9f8984c028bf59 2024-11-28T01:43:11,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/051bca60007d4874a4d492fca285e3cd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/051bca60007d4874a4d492fca285e3cd 2024-11-28T01:43:11,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71ef7a95d6a64cba99a87cb72ca01b50 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/71ef7a95d6a64cba99a87cb72ca01b50 2024-11-28T01:43:11,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d63512a9dd1f4220b4d05285594a4f07 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/d63512a9dd1f4220b4d05285594a4f07 2024-11-28T01:43:11,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/9df0c28574ee48f5a260d2a3709e65d4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/9df0c28574ee48f5a260d2a3709e65d4 2024-11-28T01:43:11,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/2360cd94d0a24a4a9adcfdc3802943f4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/2360cd94d0a24a4a9adcfdc3802943f4 2024-11-28T01:43:11,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0c6e9c6879104569ad516d4a3d71a10e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0c6e9c6879104569ad516d4a3d71a10e 2024-11-28T01:43:11,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/631eaaaa99dd42f69bdc892c4917ac1b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/631eaaaa99dd42f69bdc892c4917ac1b 2024-11-28T01:43:11,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/10aa8d5075f1432f9e3d066c03b3a180 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/10aa8d5075f1432f9e3d066c03b3a180 2024-11-28T01:43:11,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/799c618572984230aa3a96843617afe4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/799c618572984230aa3a96843617afe4 2024-11-28T01:43:11,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b9a5809ac57442b4918ea7715e62d50e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/b9a5809ac57442b4918ea7715e62d50e 2024-11-28T01:43:11,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/1249499801604d01ba31fc3a2cd38d57 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/1249499801604d01ba31fc3a2cd38d57 2024-11-28T01:43:11,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/72fed8bc752843f2a9d6bd99070fd073, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/398fe6f0936b4c86b95004b36ec9a5fe, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/22f35b9b9ab64176af645962f9463ea4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/900d8e2d7cea49c59e6d8580cbe92f85, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/70bf5a23acd8439da54c0253eb580bc8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/ae02c2eda1b14787884f5b96623a87ec, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c94ea028846f4e398414caecf6bc3762, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/369d3189773f4afc8bff88072b6a0386, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4fc44dd67d614751acc14204bd15b560, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/2834f6a4cf424228a2344b69ed46e2a9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/797a857304844fb8bfbec6f807e9198d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8c9e30c2013a47bd8e8e5eda4f0aa01c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/77447392756c4cb8a88bc834ff84b7e9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/b6a7daf66c994330ac079199bfe1325f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c3b6eb766a0141d69ab836a8c3773fd0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a0bf76c7299440f5826a9c61ad59b089, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a2971dbac1484529bda4ce359947eb96, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/27005d0e11c840fdb6edae447f4768a9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1b918f27ffdb4b9abfa0c8f23c2caada, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/20565542fa8344d6be03801e0fc3cf93, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d687b7ea932a47c9bde0346b55a1e175, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/25007ff03d0747cb9b3f5cb857995baf, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/600a7770d38240bf8c3284095a7ac9e4, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/aa720cf184bb446bbd146bff7484bcc8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d24511162c6140e7a14f22d4480ed8da, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d9a0f928c4d74d2b82b38b114dba3fe7] to archive 2024-11-28T01:43:11,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:43:11,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/72fed8bc752843f2a9d6bd99070fd073 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/72fed8bc752843f2a9d6bd99070fd073 2024-11-28T01:43:11,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/398fe6f0936b4c86b95004b36ec9a5fe to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/398fe6f0936b4c86b95004b36ec9a5fe 2024-11-28T01:43:11,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/22f35b9b9ab64176af645962f9463ea4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/22f35b9b9ab64176af645962f9463ea4 2024-11-28T01:43:11,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/900d8e2d7cea49c59e6d8580cbe92f85 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/900d8e2d7cea49c59e6d8580cbe92f85 2024-11-28T01:43:11,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/70bf5a23acd8439da54c0253eb580bc8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/70bf5a23acd8439da54c0253eb580bc8 2024-11-28T01:43:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/ae02c2eda1b14787884f5b96623a87ec to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/ae02c2eda1b14787884f5b96623a87ec 2024-11-28T01:43:11,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c94ea028846f4e398414caecf6bc3762 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c94ea028846f4e398414caecf6bc3762 2024-11-28T01:43:11,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/369d3189773f4afc8bff88072b6a0386 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/369d3189773f4afc8bff88072b6a0386 2024-11-28T01:43:11,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4fc44dd67d614751acc14204bd15b560 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/4fc44dd67d614751acc14204bd15b560 2024-11-28T01:43:11,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/2834f6a4cf424228a2344b69ed46e2a9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/2834f6a4cf424228a2344b69ed46e2a9 2024-11-28T01:43:11,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/797a857304844fb8bfbec6f807e9198d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/797a857304844fb8bfbec6f807e9198d 2024-11-28T01:43:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8c9e30c2013a47bd8e8e5eda4f0aa01c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/8c9e30c2013a47bd8e8e5eda4f0aa01c 2024-11-28T01:43:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/77447392756c4cb8a88bc834ff84b7e9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/77447392756c4cb8a88bc834ff84b7e9 2024-11-28T01:43:11,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/b6a7daf66c994330ac079199bfe1325f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/b6a7daf66c994330ac079199bfe1325f 2024-11-28T01:43:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c3b6eb766a0141d69ab836a8c3773fd0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/c3b6eb766a0141d69ab836a8c3773fd0 2024-11-28T01:43:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a0bf76c7299440f5826a9c61ad59b089 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a0bf76c7299440f5826a9c61ad59b089 2024-11-28T01:43:11,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a2971dbac1484529bda4ce359947eb96 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/a2971dbac1484529bda4ce359947eb96 2024-11-28T01:43:11,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/27005d0e11c840fdb6edae447f4768a9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/27005d0e11c840fdb6edae447f4768a9 2024-11-28T01:43:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1b918f27ffdb4b9abfa0c8f23c2caada to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1b918f27ffdb4b9abfa0c8f23c2caada 2024-11-28T01:43:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/20565542fa8344d6be03801e0fc3cf93 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/20565542fa8344d6be03801e0fc3cf93 2024-11-28T01:43:11,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d687b7ea932a47c9bde0346b55a1e175 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d687b7ea932a47c9bde0346b55a1e175 2024-11-28T01:43:11,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/25007ff03d0747cb9b3f5cb857995baf to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/25007ff03d0747cb9b3f5cb857995baf 2024-11-28T01:43:11,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/600a7770d38240bf8c3284095a7ac9e4 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/600a7770d38240bf8c3284095a7ac9e4 2024-11-28T01:43:11,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/aa720cf184bb446bbd146bff7484bcc8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/aa720cf184bb446bbd146bff7484bcc8 2024-11-28T01:43:11,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d24511162c6140e7a14f22d4480ed8da to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d24511162c6140e7a14f22d4480ed8da 2024-11-28T01:43:11,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d9a0f928c4d74d2b82b38b114dba3fe7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/d9a0f928c4d74d2b82b38b114dba3fe7 2024-11-28T01:43:11,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/66259ca9254a49faa48913bf34c6e2b1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e132526f07a8456cb8705a93c952e22c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/7b9b7d4a00bf46f48f17691143373f0b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/44b23a41b6a743f18e3d59cd165c2c0d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/118ebdab152a4635af91d29146e95d7b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/5e89e390ac0f42c3b5c5099ecbeb7153, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f9bf8e3df2fb4a56985028d265ab3d07, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020dc3875fc3456fa7d5a7ec76bd5b7b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2581ad9c073b49808292aaf9c8aa0088, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/79e5aece83b84c7ebc9d201f08f30356, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/569892c3fe4d4bc8b222618420c26f2a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/4431bbcffa634cd6950117b0195df9d9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/20bc805c8f584d3d881da28dba20c27f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/d36ba121e2484d41b11043195616be68, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/bf2c7bedacfb4e85a454c0bd71716e3a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ec268f2867dc44198cfc421c94008544, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9ece6fd6943441c48505dce9f195eda0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/0638bd85aa4e4d4fabcda81bc9238b3a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/76884d7866df4e68b1cdda1d3d8a789e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ea8ac89dcb9243e9b26fb75a5da6fcb1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e937c19e26eb48f091a9f1f103513476, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020d5ecd4d6e4a1b8ee5e3e3ffd56598, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/927e1768584644fd89728f0e91a01770, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f7e3041091dd4c8b9ca2403f94f1950b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/3d9f9c091e8d4663b83d045a792a514c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/1c92c1a11c554abe989a478a57fe589e] to archive 2024-11-28T01:43:11,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:43:11,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/66259ca9254a49faa48913bf34c6e2b1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/66259ca9254a49faa48913bf34c6e2b1 2024-11-28T01:43:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e132526f07a8456cb8705a93c952e22c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e132526f07a8456cb8705a93c952e22c 2024-11-28T01:43:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/7b9b7d4a00bf46f48f17691143373f0b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/7b9b7d4a00bf46f48f17691143373f0b 2024-11-28T01:43:11,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/44b23a41b6a743f18e3d59cd165c2c0d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/44b23a41b6a743f18e3d59cd165c2c0d 2024-11-28T01:43:11,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/118ebdab152a4635af91d29146e95d7b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/118ebdab152a4635af91d29146e95d7b 2024-11-28T01:43:11,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/5e89e390ac0f42c3b5c5099ecbeb7153 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/5e89e390ac0f42c3b5c5099ecbeb7153 2024-11-28T01:43:11,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f9bf8e3df2fb4a56985028d265ab3d07 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f9bf8e3df2fb4a56985028d265ab3d07 2024-11-28T01:43:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020dc3875fc3456fa7d5a7ec76bd5b7b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020dc3875fc3456fa7d5a7ec76bd5b7b 2024-11-28T01:43:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2581ad9c073b49808292aaf9c8aa0088 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2581ad9c073b49808292aaf9c8aa0088 2024-11-28T01:43:11,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/79e5aece83b84c7ebc9d201f08f30356 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/79e5aece83b84c7ebc9d201f08f30356 2024-11-28T01:43:11,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/569892c3fe4d4bc8b222618420c26f2a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/569892c3fe4d4bc8b222618420c26f2a 2024-11-28T01:43:11,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/4431bbcffa634cd6950117b0195df9d9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/4431bbcffa634cd6950117b0195df9d9 2024-11-28T01:43:11,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/20bc805c8f584d3d881da28dba20c27f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/20bc805c8f584d3d881da28dba20c27f 2024-11-28T01:43:11,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/d36ba121e2484d41b11043195616be68 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/d36ba121e2484d41b11043195616be68 2024-11-28T01:43:11,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/bf2c7bedacfb4e85a454c0bd71716e3a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/bf2c7bedacfb4e85a454c0bd71716e3a 2024-11-28T01:43:11,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ec268f2867dc44198cfc421c94008544 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ec268f2867dc44198cfc421c94008544 2024-11-28T01:43:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9ece6fd6943441c48505dce9f195eda0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9ece6fd6943441c48505dce9f195eda0 2024-11-28T01:43:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/0638bd85aa4e4d4fabcda81bc9238b3a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/0638bd85aa4e4d4fabcda81bc9238b3a 2024-11-28T01:43:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/76884d7866df4e68b1cdda1d3d8a789e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/76884d7866df4e68b1cdda1d3d8a789e 2024-11-28T01:43:11,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ea8ac89dcb9243e9b26fb75a5da6fcb1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ea8ac89dcb9243e9b26fb75a5da6fcb1 2024-11-28T01:43:11,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e937c19e26eb48f091a9f1f103513476 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/e937c19e26eb48f091a9f1f103513476 2024-11-28T01:43:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020d5ecd4d6e4a1b8ee5e3e3ffd56598 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/020d5ecd4d6e4a1b8ee5e3e3ffd56598 2024-11-28T01:43:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/927e1768584644fd89728f0e91a01770 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/927e1768584644fd89728f0e91a01770 2024-11-28T01:43:11,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f7e3041091dd4c8b9ca2403f94f1950b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/f7e3041091dd4c8b9ca2403f94f1950b 2024-11-28T01:43:11,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/3d9f9c091e8d4663b83d045a792a514c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/3d9f9c091e8d4663b83d045a792a514c 2024-11-28T01:43:11,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/1c92c1a11c554abe989a478a57fe589e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/1c92c1a11c554abe989a478a57fe589e 2024-11-28T01:43:11,168 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/recovered.edits/465.seqid, newMaxSeqId=465, maxSeqId=1 2024-11-28T01:43:11,169 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. 2024-11-28T01:43:11,169 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 9f37285715eee60df10bafa1eb0d60a0: Waiting for close lock at 1732758189856Running coprocessor pre-close hooks at 1732758189856Disabling compacts and flushes for region at 1732758189856Disabling writes for close at 1732758189856Obtaining lock to block concurrent updates at 1732758189856Preparing flush snapshotting stores in 9f37285715eee60df10bafa1eb0d60a0 at 1732758189856Finished memstore snapshotting TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732758189856Flushing stores of TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0. at 1732758189857 (+1 ms)Flushing 9f37285715eee60df10bafa1eb0d60a0/A: creating writer at 1732758189857Flushing 9f37285715eee60df10bafa1eb0d60a0/A: appending metadata at 1732758189859 (+2 ms)Flushing 9f37285715eee60df10bafa1eb0d60a0/A: closing flushed file at 1732758189859Flushing 9f37285715eee60df10bafa1eb0d60a0/B: creating writer at 1732758190267 (+408 ms)Flushing 9f37285715eee60df10bafa1eb0d60a0/B: appending metadata at 1732758190269 (+2 ms)Flushing 9f37285715eee60df10bafa1eb0d60a0/B: closing flushed file at 1732758190269Flushing 9f37285715eee60df10bafa1eb0d60a0/C: creating writer at 1732758190676 (+407 ms)Flushing 9f37285715eee60df10bafa1eb0d60a0/C: appending metadata at 1732758190679 (+3 ms)Flushing 9f37285715eee60df10bafa1eb0d60a0/C: closing flushed file at 1732758190679Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76307258: reopening flushed file at 1732758191086 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b814e76: reopening flushed file at 1732758191089 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c6fce71: reopening flushed file at 1732758191092 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 9f37285715eee60df10bafa1eb0d60a0 in 1240ms, sequenceid=462, compaction requested=true at 1732758191096 (+4 ms)Writing region close event to WAL at 1732758191166 (+70 ms)Running coprocessor post-close hooks at 1732758191169 (+3 ms)Closed at 1732758191169 2024-11-28T01:43:11,171 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:11,171 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=9f37285715eee60df10bafa1eb0d60a0, regionState=CLOSED 2024-11-28T01:43:11,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:43:11,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-28T01:43:11,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 9f37285715eee60df10bafa1eb0d60a0, server=9df3f25f5867,42371,1732758050017 in 1.4700 sec 2024-11-28T01:43:11,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-28T01:43:11,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9f37285715eee60df10bafa1eb0d60a0, UNASSIGN in 1.4730 sec 2024-11-28T01:43:11,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-28T01:43:11,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4760 sec 2024-11-28T01:43:11,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758191178"}]},"ts":"1732758191178"} 2024-11-28T01:43:11,180 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T01:43:11,180 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T01:43:11,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4840 sec 2024-11-28T01:43:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-28T01:43:11,843 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:11,843 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T01:43:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:11,845 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T01:43:11,846 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:11,848 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:11,850 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/recovered.edits] 2024-11-28T01:43:11,852 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0aa2afc203e84ec9ac793b073090e04d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/0aa2afc203e84ec9ac793b073090e04d 2024-11-28T01:43:11,853 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/4ef76cdaeced4095997b4114001c1b1d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/4ef76cdaeced4095997b4114001c1b1d 2024-11-28T01:43:11,854 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a20a3d995c25403ba5bb61a38e0779ef to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/A/a20a3d995c25403ba5bb61a38e0779ef 2024-11-28T01:43:11,856 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1545655572a24749b6752845cfea7fcc to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/1545655572a24749b6752845cfea7fcc 2024-11-28T01:43:11,857 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/67b901fb40fc45f49bae3938d1d46af1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/67b901fb40fc45f49bae3938d1d46af1 2024-11-28T01:43:11,858 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/7ac20511bf974bf998f17b45ae5ca63f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/B/7ac20511bf974bf998f17b45ae5ca63f 2024-11-28T01:43:11,860 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2f8ab65caa1443a0a9b07f3772f8c874 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/2f8ab65caa1443a0a9b07f3772f8c874 2024-11-28T01:43:11,861 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9b836a3fcb6045899f1d5f8af724a819 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/9b836a3fcb6045899f1d5f8af724a819 2024-11-28T01:43:11,862 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ee6d5111887f4c43af7464cf09864402 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/C/ee6d5111887f4c43af7464cf09864402 2024-11-28T01:43:11,864 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/recovered.edits/465.seqid to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0/recovered.edits/465.seqid 2024-11-28T01:43:11,864 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/9f37285715eee60df10bafa1eb0d60a0 2024-11-28T01:43:11,864 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T01:43:11,866 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:11,868 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T01:43:11,869 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T01:43:11,870 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:11,870 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T01:43:11,871 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732758191870"}]},"ts":"9223372036854775807"} 2024-11-28T01:43:11,872 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T01:43:11,872 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9f37285715eee60df10bafa1eb0d60a0, NAME => 'TestAcidGuarantees,,1732758164823.9f37285715eee60df10bafa1eb0d60a0.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T01:43:11,872 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T01:43:11,872 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732758191872"}]},"ts":"9223372036854775807"} 2024-11-28T01:43:11,873 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T01:43:11,874 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:11,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-28T01:43:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-28T01:43:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T01:43:11,953 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:11,963 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 251), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=566 (was 639), ProcessCount=11 (was 11), AvailableMemoryMB=6582 (was 6627) 2024-11-28T01:43:11,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=566, ProcessCount=11, AvailableMemoryMB=6582 2024-11-28T01:43:11,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:43:11,973 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:43:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:11,975 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:43:11,975 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:11,975 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-28T01:43:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T01:43:11,976 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:43:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742438_1614 (size=1061) 2024-11-28T01:43:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T01:43:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T01:43:12,382 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec 2024-11-28T01:43:12,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742439_1615 (size=53) 2024-11-28T01:43:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T01:43:12,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:43:12,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b33d90fd8529f8494aea715fa8bbbcde, disabling compactions & flushes 2024-11-28T01:43:12,788 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:12,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:12,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. after waiting 0 ms 2024-11-28T01:43:12,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:12,788 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:12,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b33d90fd8529f8494aea715fa8bbbcde: Waiting for close lock at 1732758192788Disabling compacts and flushes for region at 1732758192788Disabling writes for close at 1732758192788Writing region close event to WAL at 1732758192788Closed at 1732758192788 2024-11-28T01:43:12,789 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:43:12,789 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732758192789"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758192789"}]},"ts":"1732758192789"} 2024-11-28T01:43:12,791 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:43:12,791 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:43:12,791 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758192791"}]},"ts":"1732758192791"} 2024-11-28T01:43:12,792 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T01:43:12,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, ASSIGN}] 2024-11-28T01:43:12,793 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, ASSIGN 2024-11-28T01:43:12,794 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, ASSIGN; state=OFFLINE, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=false 2024-11-28T01:43:12,944 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:12,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, ASSIGN because future has completed 2024-11-28T01:43:12,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:43:13,100 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:13,100 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:43:13,101 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,101 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:43:13,101 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,101 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,102 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,103 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:43:13,103 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b33d90fd8529f8494aea715fa8bbbcde columnFamilyName A 2024-11-28T01:43:13,103 DEBUG [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:13,104 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(327): Store=b33d90fd8529f8494aea715fa8bbbcde/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:43:13,104 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,105 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:43:13,105 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b33d90fd8529f8494aea715fa8bbbcde columnFamilyName B 2024-11-28T01:43:13,105 DEBUG [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:13,105 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(327): Store=b33d90fd8529f8494aea715fa8bbbcde/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:43:13,105 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,106 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:43:13,106 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b33d90fd8529f8494aea715fa8bbbcde columnFamilyName C 2024-11-28T01:43:13,106 DEBUG [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:13,107 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(327): Store=b33d90fd8529f8494aea715fa8bbbcde/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:43:13,107 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:13,107 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,107 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,108 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,109 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,109 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,109 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:43:13,110 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,111 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:43:13,111 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened b33d90fd8529f8494aea715fa8bbbcde; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60851793, jitterRate=-0.09323762357234955}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:43:13,112 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:13,112 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for b33d90fd8529f8494aea715fa8bbbcde: Running coprocessor pre-open hook at 1732758193101Writing region info on filesystem at 1732758193101Initializing all the Stores at 1732758193102 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758193102Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758193102Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758193102Cleaning up temporary data from old regions at 1732758193109 (+7 ms)Running coprocessor post-open hooks at 1732758193112 (+3 ms)Region opened successfully at 1732758193112 2024-11-28T01:43:13,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T01:43:13,113 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., pid=148, masterSystemTime=1732758193098 2024-11-28T01:43:13,114 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:13,114 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:13,115 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:13,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:43:13,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-28T01:43:13,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 in 171 msec 2024-11-28T01:43:13,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-28T01:43:13,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, ASSIGN in 325 msec 2024-11-28T01:43:13,120 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:43:13,120 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758193120"}]},"ts":"1732758193120"} 2024-11-28T01:43:13,122 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T01:43:13,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:43:13,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-28T01:43:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-28T01:43:14,123 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:14,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b31ef7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:14,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:14,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:14,125 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f368bfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:14,126 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e833343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:14,127 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:14,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:14,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:43:14,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,45903,1732758049085 2024-11-28T01:43:14,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@535d93ff 2024-11-28T01:43:14,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:43:14,131 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:43:14,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T01:43:14,132 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:43:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:14,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742440_1616 (size=1097) 2024-11-28T01:43:14,543 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T01:43:14,543 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T01:43:14,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:43:14,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, REOPEN/MOVE}] 2024-11-28T01:43:14,548 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, REOPEN/MOVE 2024-11-28T01:43:14,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:14,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, REOPEN/MOVE because future has completed 2024-11-28T01:43:14,550 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:43:14,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:43:14,702 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:14,702 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:43:14,702 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing b33d90fd8529f8494aea715fa8bbbcde, disabling compactions & flushes 2024-11-28T01:43:14,702 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:14,702 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:14,702 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. after waiting 0 ms 2024-11-28T01:43:14,702 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:14,705 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T01:43:14,706 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:14,706 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for b33d90fd8529f8494aea715fa8bbbcde: Waiting for close lock at 1732758194702Running coprocessor pre-close hooks at 1732758194702Disabling compacts and flushes for region at 1732758194702Disabling writes for close at 1732758194702Writing region close event to WAL at 1732758194702Running coprocessor post-close hooks at 1732758194706 (+4 ms)Closed at 1732758194706 2024-11-28T01:43:14,706 WARN [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: b33d90fd8529f8494aea715fa8bbbcde to self. 2024-11-28T01:43:14,707 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:14,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=CLOSED 2024-11-28T01:43:14,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:43:14,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-28T01:43:14,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 in 160 msec 2024-11-28T01:43:14,711 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, REOPEN/MOVE; state=CLOSED, location=9df3f25f5867,42371,1732758050017; forceNewPlan=false, retain=true 2024-11-28T01:43:14,862 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=OPENING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:14,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, REOPEN/MOVE because future has completed 2024-11-28T01:43:14,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:43:15,018 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,018 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:43:15,018 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,019 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:43:15,019 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,019 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,020 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,020 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:43:15,021 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b33d90fd8529f8494aea715fa8bbbcde columnFamilyName A 2024-11-28T01:43:15,022 DEBUG [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:15,022 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(327): Store=b33d90fd8529f8494aea715fa8bbbcde/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:43:15,022 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,023 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:43:15,023 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b33d90fd8529f8494aea715fa8bbbcde columnFamilyName B 2024-11-28T01:43:15,023 DEBUG [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:15,023 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(327): Store=b33d90fd8529f8494aea715fa8bbbcde/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:43:15,023 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,024 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T01:43:15,024 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b33d90fd8529f8494aea715fa8bbbcde columnFamilyName C 2024-11-28T01:43:15,024 DEBUG [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:15,024 INFO [StoreOpener-b33d90fd8529f8494aea715fa8bbbcde-1 {}] regionserver.HStore(327): Store=b33d90fd8529f8494aea715fa8bbbcde/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:43:15,024 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,024 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,025 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,026 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,027 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,027 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,027 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T01:43:15,028 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,029 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened b33d90fd8529f8494aea715fa8bbbcde; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72383150, jitterRate=0.07859298586845398}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T01:43:15,029 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,030 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for b33d90fd8529f8494aea715fa8bbbcde: Running coprocessor pre-open hook at 1732758195019Writing region info on filesystem at 1732758195019Initializing all the Stores at 1732758195019Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758195019Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758195020 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758195020Cleaning up temporary data from old regions at 1732758195027 (+7 ms)Running coprocessor post-open hooks at 1732758195029 (+2 ms)Region opened successfully at 1732758195030 (+1 ms) 2024-11-28T01:43:15,030 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., pid=153, masterSystemTime=1732758195015 2024-11-28T01:43:15,032 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,032 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,033 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=OPEN, openSeqNum=5, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:43:15,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-28T01:43:15,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 in 171 msec 2024-11-28T01:43:15,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-28T01:43:15,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, REOPEN/MOVE in 489 msec 2024-11-28T01:43:15,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-28T01:43:15,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-28T01:43:15,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-28T01:43:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-28T01:43:15,042 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:15,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b12efd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c122a08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,047 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354565a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,049 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e48ada1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,050 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f0fec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d69e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff3615c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,054 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af5588e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1270f48d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,057 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce9c527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,059 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4e8a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654a17ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,062 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a32b20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,064 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f8b192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,066 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@285084fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,067 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b47608, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,073 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d205cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,074 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575254a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,45903,-1 for getting cluster id 2024-11-28T01:43:15,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:43:15,076 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb45315c-53b7-46a3-81e7-db91cf2c055a' 2024-11-28T01:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb45315c-53b7-46a3-81e7-db91cf2c055a" 2024-11-28T01:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72461fcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,45903,-1] 2024-11-28T01:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:43:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:15,077 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:43:15,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10341f38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:43:15,078 DEBUG [Thread-2723 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,078 DEBUG [Thread-2725 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,078 DEBUG [Thread-2708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,078 DEBUG [Thread-2714 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,079 DEBUG [Thread-2710 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,079 DEBUG [Thread-2706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,079 DEBUG [Thread-2719 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,079 DEBUG [Thread-2712 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,080 DEBUG [Thread-2721 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,080 DEBUG [Thread-2717 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:43:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:15,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,083 DEBUG [Thread-2717 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,42371,1732758050017, seqNum=-1] 2024-11-28T01:43:15,083 DEBUG [Thread-2717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:43:15,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-28T01:43:15,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T01:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,087 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:15,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5] 2024-11-28T01:43:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:43:15,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:15,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:15,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:15,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:15,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:15,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:15,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a3088ab701b44adcb60c49d821f8ce1b_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_1/A:col10/1732758195133/Put/seqid=0 2024-11-28T01:43:15,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758255156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758255156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758255158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758195154, max timestamp=1732758195160Num uniques 859; 2024-11-28T01:43:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758195154, max timestamp=1732758195160Num uniques 859; 2024-11-28T01:43:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:43:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:43:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758195154, max timestamp=1732758195160Num uniques 859; 2024-11-28T01:43:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:43:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742441_1617 (size=9714) 2024-11-28T01:43:15,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:15,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a3088ab701b44adcb60c49d821f8ce1b_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a3088ab701b44adcb60c49d821f8ce1b_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:15,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/24f1cce9f033475194bbffe6b95a9337, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:15,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/24f1cce9f033475194bbffe6b95a9337 is 175, key is test_row_1/A:col10/1732758195133/Put/seqid=0 2024-11-28T01:43:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T01:43:15,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742442_1618 (size=22361) 2024-11-28T01:43:15,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758255202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758255203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T01:43:15,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:15,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758255273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758255273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758255273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758255312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758255312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T01:43:15,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:15,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T01:43:15,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758255482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758255482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758255482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758255523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758255523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T01:43:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/24f1cce9f033475194bbffe6b95a9337 2024-11-28T01:43:15,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/2a8d65f6560745eb97fe98390840f719 is 50, key is test_row_1/B:col10/1732758195133/Put/seqid=0 2024-11-28T01:43:15,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742443_1619 (size=9657) 2024-11-28T01:43:15,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T01:43:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T01:43:15,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758255792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758255792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758255792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758255833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758255833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:15,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T01:43:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:15,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:16,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T01:43:16,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:16,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:16,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:16,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/2a8d65f6560745eb97fe98390840f719 2024-11-28T01:43:16,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/4f2c4f788a614e27b2c44485d1009f3b is 50, key is test_row_1/C:col10/1732758195133/Put/seqid=0 2024-11-28T01:43:16,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742444_1620 (size=9657) 2024-11-28T01:43:16,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/4f2c4f788a614e27b2c44485d1009f3b 2024-11-28T01:43:16,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/24f1cce9f033475194bbffe6b95a9337 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/24f1cce9f033475194bbffe6b95a9337 2024-11-28T01:43:16,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/24f1cce9f033475194bbffe6b95a9337, entries=100, sequenceid=15, filesize=21.8 K 2024-11-28T01:43:16,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/2a8d65f6560745eb97fe98390840f719 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/2a8d65f6560745eb97fe98390840f719 2024-11-28T01:43:16,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/2a8d65f6560745eb97fe98390840f719, entries=100, sequenceid=15, filesize=9.4 K 2024-11-28T01:43:16,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/4f2c4f788a614e27b2c44485d1009f3b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4f2c4f788a614e27b2c44485d1009f3b 2024-11-28T01:43:16,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4f2c4f788a614e27b2c44485d1009f3b, entries=100, sequenceid=15, filesize=9.4 K 2024-11-28T01:43:16,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b33d90fd8529f8494aea715fa8bbbcde in 957ms, sequenceid=15, compaction requested=false 2024-11-28T01:43:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:16,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T01:43:16,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:16,158 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-28T01:43:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:16,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c09fe669a74f445fa2aa92060b937d2a_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758195201/Put/seqid=0 2024-11-28T01:43:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742445_1621 (size=12154) 2024-11-28T01:43:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T01:43:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:16,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758256313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758256314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758256317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758256342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758256342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758256422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758256422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758256422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:16,573 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c09fe669a74f445fa2aa92060b937d2a_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c09fe669a74f445fa2aa92060b937d2a_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:16,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/18cf0dd0c21d4005aef1a20bc48b5dd0, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:16,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/18cf0dd0c21d4005aef1a20bc48b5dd0 is 175, key is test_row_0/A:col10/1732758195201/Put/seqid=0 2024-11-28T01:43:16,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742446_1622 (size=30955) 2024-11-28T01:43:16,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758256633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758256633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758256633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,688 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:43:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758256942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758256942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758256943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:16,981 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/18cf0dd0c21d4005aef1a20bc48b5dd0 2024-11-28T01:43:16,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/1d5ff9a2c4da425aa1e8dd1999c92dc1 is 50, key is test_row_0/B:col10/1732758195201/Put/seqid=0 2024-11-28T01:43:17,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742447_1623 (size=12001) 2024-11-28T01:43:17,001 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/1d5ff9a2c4da425aa1e8dd1999c92dc1 2024-11-28T01:43:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/4e48adc9f1ca4d9381b6c4c635bbd40f is 50, key is test_row_0/C:col10/1732758195201/Put/seqid=0 2024-11-28T01:43:17,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742448_1624 (size=12001) 2024-11-28T01:43:17,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/4e48adc9f1ca4d9381b6c4c635bbd40f 2024-11-28T01:43:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/18cf0dd0c21d4005aef1a20bc48b5dd0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/18cf0dd0c21d4005aef1a20bc48b5dd0 2024-11-28T01:43:17,036 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/18cf0dd0c21d4005aef1a20bc48b5dd0, entries=150, sequenceid=51, filesize=30.2 K 2024-11-28T01:43:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/1d5ff9a2c4da425aa1e8dd1999c92dc1 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1d5ff9a2c4da425aa1e8dd1999c92dc1 2024-11-28T01:43:17,040 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1d5ff9a2c4da425aa1e8dd1999c92dc1, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T01:43:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/4e48adc9f1ca4d9381b6c4c635bbd40f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4e48adc9f1ca4d9381b6c4c635bbd40f 2024-11-28T01:43:17,050 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4e48adc9f1ca4d9381b6c4c635bbd40f, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T01:43:17,051 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for b33d90fd8529f8494aea715fa8bbbcde in 893ms, sequenceid=51, compaction requested=false 2024-11-28T01:43:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-28T01:43:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-28T01:43:17,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-28T01:43:17,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9660 sec 2024-11-28T01:43:17,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.9740 sec 2024-11-28T01:43:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T01:43:17,233 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:17,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-28T01:43:17,236 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T01:43:17,237 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:17,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T01:43:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:17,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T01:43:17,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:17,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:17,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:17,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:17,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:17,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:17,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112865e16c7c05f34a9881f0866c1a17c71d_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758196308/Put/seqid=0 2024-11-28T01:43:17,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742449_1625 (size=14594) 2024-11-28T01:43:17,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T01:43:17,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758257397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758257401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758257452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758257452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758257452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758257512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758257512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T01:43:17,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T01:43:17,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T01:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758257722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758257722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:17,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:17,777 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112865e16c7c05f34a9881f0866c1a17c71d_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112865e16c7c05f34a9881f0866c1a17c71d_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:17,777 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/fb26fc3375c14939a05a9cf6d61127e3, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:17,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/fb26fc3375c14939a05a9cf6d61127e3 is 175, key is test_row_0/A:col10/1732758196308/Put/seqid=0 2024-11-28T01:43:17,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742450_1626 (size=39549) 2024-11-28T01:43:17,783 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/fb26fc3375c14939a05a9cf6d61127e3 2024-11-28T01:43:17,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/a6e3f5e6c3954a4a931b50a8b180c267 is 50, key is test_row_0/B:col10/1732758196308/Put/seqid=0 2024-11-28T01:43:17,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742451_1627 (size=12001) 2024-11-28T01:43:17,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T01:43:17,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:17,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:17,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T01:43:18,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T01:43:18,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:18,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:18,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:18,020 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T01:43:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758258032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758258033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T01:43:18,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:18,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:18,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:18,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:18,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/a6e3f5e6c3954a4a931b50a8b180c267 2024-11-28T01:43:18,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3751d5ec080f468a99a4854ac4c71729 is 50, key is test_row_0/C:col10/1732758196308/Put/seqid=0 2024-11-28T01:43:18,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742452_1628 (size=12001) 2024-11-28T01:43:18,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3751d5ec080f468a99a4854ac4c71729 2024-11-28T01:43:18,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/fb26fc3375c14939a05a9cf6d61127e3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/fb26fc3375c14939a05a9cf6d61127e3 2024-11-28T01:43:18,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/fb26fc3375c14939a05a9cf6d61127e3, entries=200, sequenceid=63, filesize=38.6 K 2024-11-28T01:43:18,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/a6e3f5e6c3954a4a931b50a8b180c267 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a6e3f5e6c3954a4a931b50a8b180c267 2024-11-28T01:43:18,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a6e3f5e6c3954a4a931b50a8b180c267, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T01:43:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3751d5ec080f468a99a4854ac4c71729 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3751d5ec080f468a99a4854ac4c71729 2024-11-28T01:43:18,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3751d5ec080f468a99a4854ac4c71729, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T01:43:18,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b33d90fd8529f8494aea715fa8bbbcde in 875ms, sequenceid=63, compaction requested=true 2024-11-28T01:43:18,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:18,230 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:18,231 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:18,231 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:18,231 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,231 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/24f1cce9f033475194bbffe6b95a9337, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/18cf0dd0c21d4005aef1a20bc48b5dd0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/fb26fc3375c14939a05a9cf6d61127e3] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=90.7 K 2024-11-28T01:43:18,231 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,231 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/24f1cce9f033475194bbffe6b95a9337, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/18cf0dd0c21d4005aef1a20bc48b5dd0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/fb26fc3375c14939a05a9cf6d61127e3] 2024-11-28T01:43:18,232 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f1cce9f033475194bbffe6b95a9337, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758195132 2024-11-28T01:43:18,232 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18cf0dd0c21d4005aef1a20bc48b5dd0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732758195191 2024-11-28T01:43:18,233 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb26fc3375c14939a05a9cf6d61127e3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758196308 2024-11-28T01:43:18,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:18,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:18,234 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:18,235 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:18,235 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:18,235 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,235 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/2a8d65f6560745eb97fe98390840f719, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1d5ff9a2c4da425aa1e8dd1999c92dc1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a6e3f5e6c3954a4a931b50a8b180c267] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=32.9 K 2024-11-28T01:43:18,236 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8d65f6560745eb97fe98390840f719, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758195132 2024-11-28T01:43:18,236 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5ff9a2c4da425aa1e8dd1999c92dc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732758195191 2024-11-28T01:43:18,237 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a6e3f5e6c3954a4a931b50a8b180c267, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758196308 2024-11-28T01:43:18,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:18,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:18,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:18,244 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:18,246 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:18,247 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/3132e3e294c04572a2fb77dd762e0643 is 50, key is test_row_0/B:col10/1732758196308/Put/seqid=0 2024-11-28T01:43:18,264 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411284ac80849ac224a978d5d7763f3ea5ce5_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:18,266 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411284ac80849ac224a978d5d7763f3ea5ce5_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:18,267 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284ac80849ac224a978d5d7763f3ea5ce5_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742453_1629 (size=12104) 2024-11-28T01:43:18,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T01:43:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,309 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:43:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:18,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742454_1630 (size=4469) 2024-11-28T01:43:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411286d68e41d93dd4d60b45f9ab54a2ec10d_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758197400/Put/seqid=0 2024-11-28T01:43:18,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742455_1631 (size=12154) 2024-11-28T01:43:18,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:18,347 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411286d68e41d93dd4d60b45f9ab54a2ec10d_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411286d68e41d93dd4d60b45f9ab54a2ec10d_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:18,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8538732fd0a345f291588d5556e787e2, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:18,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8538732fd0a345f291588d5556e787e2 is 175, key is test_row_0/A:col10/1732758197400/Put/seqid=0 2024-11-28T01:43:18,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742456_1632 (size=30955) 2024-11-28T01:43:18,354 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8538732fd0a345f291588d5556e787e2 2024-11-28T01:43:18,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/04effb66d97847f982cc2afa3bea0eeb is 50, key is test_row_0/B:col10/1732758197400/Put/seqid=0 2024-11-28T01:43:18,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742457_1633 (size=12001) 2024-11-28T01:43:18,366 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/04effb66d97847f982cc2afa3bea0eeb 2024-11-28T01:43:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T01:43:18,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/d53a6b3a743f4cd4809f088259587259 is 50, key is test_row_0/C:col10/1732758197400/Put/seqid=0 2024-11-28T01:43:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742458_1634 (size=12001) 2024-11-28T01:43:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:18,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:18,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758258482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758258484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758258485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758258543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758258543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758258592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758258592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758258592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,711 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/3132e3e294c04572a2fb77dd762e0643 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/3132e3e294c04572a2fb77dd762e0643 2024-11-28T01:43:18,715 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into 3132e3e294c04572a2fb77dd762e0643(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:18,715 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:18,715 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758198234; duration=0sec 2024-11-28T01:43:18,715 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:18,715 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:18,715 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:18,716 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:18,716 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:18,716 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,716 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4f2c4f788a614e27b2c44485d1009f3b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4e48adc9f1ca4d9381b6c4c635bbd40f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3751d5ec080f468a99a4854ac4c71729] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=32.9 K 2024-11-28T01:43:18,716 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2c4f788a614e27b2c44485d1009f3b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732758195132 2024-11-28T01:43:18,717 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e48adc9f1ca4d9381b6c4c635bbd40f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732758195191 2024-11-28T01:43:18,717 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3751d5ec080f468a99a4854ac4c71729, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758196308 2024-11-28T01:43:18,722 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#535 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:18,723 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#540 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:18,723 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/f860bfc2b5564a15b25b873b800b3239 is 175, key is test_row_0/A:col10/1732758196308/Put/seqid=0 2024-11-28T01:43:18,723 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/77fbde1be275438d801dd9ffd6a07669 is 50, key is test_row_0/C:col10/1732758196308/Put/seqid=0 2024-11-28T01:43:18,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742459_1635 (size=31058) 2024-11-28T01:43:18,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742460_1636 (size=12104) 2024-11-28T01:43:18,732 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/77fbde1be275438d801dd9ffd6a07669 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/77fbde1be275438d801dd9ffd6a07669 2024-11-28T01:43:18,736 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into 77fbde1be275438d801dd9ffd6a07669(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:18,736 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:18,736 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758198243; duration=0sec 2024-11-28T01:43:18,736 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:18,736 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:18,777 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/d53a6b3a743f4cd4809f088259587259 2024-11-28T01:43:18,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8538732fd0a345f291588d5556e787e2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8538732fd0a345f291588d5556e787e2 2024-11-28T01:43:18,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8538732fd0a345f291588d5556e787e2, entries=150, sequenceid=83, filesize=30.2 K 2024-11-28T01:43:18,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/04effb66d97847f982cc2afa3bea0eeb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/04effb66d97847f982cc2afa3bea0eeb 2024-11-28T01:43:18,787 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/04effb66d97847f982cc2afa3bea0eeb, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T01:43:18,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/d53a6b3a743f4cd4809f088259587259 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d53a6b3a743f4cd4809f088259587259 2024-11-28T01:43:18,791 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d53a6b3a743f4cd4809f088259587259, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T01:43:18,792 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b33d90fd8529f8494aea715fa8bbbcde in 483ms, sequenceid=83, compaction requested=false 2024-11-28T01:43:18,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:18,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:18,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-28T01:43:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-28T01:43:18,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-28T01:43:18,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-28T01:43:18,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5620 sec 2024-11-28T01:43:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:18,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:43:18,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:18,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:18,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:18,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:18,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:18,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:18,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128199009a454ad4c20bf5c2295035c187a_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758198805/Put/seqid=0 2024-11-28T01:43:18,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742461_1637 (size=14594) 2024-11-28T01:43:18,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:18,821 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128199009a454ad4c20bf5c2295035c187a_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128199009a454ad4c20bf5c2295035c187a_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:18,821 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/40f13e2b8ebf4d7398886b9cf5f56bc0, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:18,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/40f13e2b8ebf4d7398886b9cf5f56bc0 is 175, key is test_row_0/A:col10/1732758198805/Put/seqid=0 2024-11-28T01:43:18,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742462_1638 (size=39549) 2024-11-28T01:43:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758258835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758258836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758258838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758258953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758258953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758258953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,131 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/f860bfc2b5564a15b25b873b800b3239 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f860bfc2b5564a15b25b873b800b3239 2024-11-28T01:43:19,135 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into f860bfc2b5564a15b25b873b800b3239(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:19,135 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:19,135 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758198230; duration=0sec 2024-11-28T01:43:19,135 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:19,135 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:19,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758259163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758259163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758259163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/40f13e2b8ebf4d7398886b9cf5f56bc0 2024-11-28T01:43:19,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/fc7d4a118e7b4260bc9292e79285f68f is 50, key is test_row_0/B:col10/1732758198805/Put/seqid=0 2024-11-28T01:43:19,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742463_1639 (size=12001) 2024-11-28T01:43:19,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/fc7d4a118e7b4260bc9292e79285f68f 2024-11-28T01:43:19,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/66426595a1d24c6293253085b238aae3 is 50, key is test_row_0/C:col10/1732758198805/Put/seqid=0 2024-11-28T01:43:19,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742464_1640 (size=12001) 2024-11-28T01:43:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T01:43:19,383 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:19,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-28T01:43:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T01:43:19,385 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:19,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:19,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758259472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758259472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758259472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T01:43:19,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T01:43:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:19,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758259552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,556 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_0' in b33d90fd8529f8494aea715fa8bbbcde of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758259563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:19,566 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in b33d90fd8529f8494aea715fa8bbbcde of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:19,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/66426595a1d24c6293253085b238aae3 2024-11-28T01:43:19,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/40f13e2b8ebf4d7398886b9cf5f56bc0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/40f13e2b8ebf4d7398886b9cf5f56bc0 2024-11-28T01:43:19,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/40f13e2b8ebf4d7398886b9cf5f56bc0, entries=200, sequenceid=99, filesize=38.6 K 2024-11-28T01:43:19,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/fc7d4a118e7b4260bc9292e79285f68f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/fc7d4a118e7b4260bc9292e79285f68f 2024-11-28T01:43:19,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/fc7d4a118e7b4260bc9292e79285f68f, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T01:43:19,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/66426595a1d24c6293253085b238aae3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/66426595a1d24c6293253085b238aae3 2024-11-28T01:43:19,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T01:43:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:19,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/66426595a1d24c6293253085b238aae3, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T01:43:19,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b33d90fd8529f8494aea715fa8bbbcde in 889ms, sequenceid=99, compaction requested=true 2024-11-28T01:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:19,695 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:19,695 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:19,696 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:19,696 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:19,696 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:19,696 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:19,696 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,696 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,696 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f860bfc2b5564a15b25b873b800b3239, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8538732fd0a345f291588d5556e787e2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/40f13e2b8ebf4d7398886b9cf5f56bc0] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=99.2 K 2024-11-28T01:43:19,696 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/3132e3e294c04572a2fb77dd762e0643, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/04effb66d97847f982cc2afa3bea0eeb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/fc7d4a118e7b4260bc9292e79285f68f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=35.3 K 2024-11-28T01:43:19,696 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,696 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f860bfc2b5564a15b25b873b800b3239, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8538732fd0a345f291588d5556e787e2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/40f13e2b8ebf4d7398886b9cf5f56bc0] 2024-11-28T01:43:19,697 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3132e3e294c04572a2fb77dd762e0643, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758196308 2024-11-28T01:43:19,697 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 04effb66d97847f982cc2afa3bea0eeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758197382 2024-11-28T01:43:19,697 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f860bfc2b5564a15b25b873b800b3239, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758196308 2024-11-28T01:43:19,698 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7d4a118e7b4260bc9292e79285f68f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758198484 2024-11-28T01:43:19,698 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8538732fd0a345f291588d5556e787e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758197382 2024-11-28T01:43:19,698 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40f13e2b8ebf4d7398886b9cf5f56bc0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758198476 2024-11-28T01:43:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T01:43:19,703 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:19,704 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#544 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:19,705 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f48e3fdcced840cebf74e99068342cdb is 50, key is test_row_0/B:col10/1732758198805/Put/seqid=0 2024-11-28T01:43:19,705 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128a9787264bf674d7ab8bbedf42c80ca89_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:19,707 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128a9787264bf674d7ab8bbedf42c80ca89_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:19,707 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a9787264bf674d7ab8bbedf42c80ca89_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:19,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742466_1642 (size=4469) 2024-11-28T01:43:19,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742465_1641 (size=12207) 2024-11-28T01:43:19,753 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f48e3fdcced840cebf74e99068342cdb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f48e3fdcced840cebf74e99068342cdb 2024-11-28T01:43:19,758 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into f48e3fdcced840cebf74e99068342cdb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:19,758 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:19,758 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758199695; duration=0sec 2024-11-28T01:43:19,758 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:19,759 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:19,759 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:19,760 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:19,760 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:19,760 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,760 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/77fbde1be275438d801dd9ffd6a07669, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d53a6b3a743f4cd4809f088259587259, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/66426595a1d24c6293253085b238aae3] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=35.3 K 2024-11-28T01:43:19,760 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 77fbde1be275438d801dd9ffd6a07669, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732758196308 2024-11-28T01:43:19,761 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d53a6b3a743f4cd4809f088259587259, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732758197382 2024-11-28T01:43:19,761 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 66426595a1d24c6293253085b238aae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758198484 2024-11-28T01:43:19,767 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#546 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:19,767 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/c5da3882788e42aca1bd90e48cc1968d is 50, key is test_row_0/C:col10/1732758198805/Put/seqid=0 2024-11-28T01:43:19,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742467_1643 (size=12207) 2024-11-28T01:43:19,781 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/c5da3882788e42aca1bd90e48cc1968d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c5da3882788e42aca1bd90e48cc1968d 2024-11-28T01:43:19,786 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into c5da3882788e42aca1bd90e48cc1968d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:19,786 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:19,786 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758199695; duration=0sec 2024-11-28T01:43:19,786 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:19,786 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:19,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T01:43:19,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:19,846 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T01:43:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128424c649a375d485fb1a6575f67378a09_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758198836/Put/seqid=0 2024-11-28T01:43:19,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742468_1644 (size=12154) 2024-11-28T01:43:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:19,861 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128424c649a375d485fb1a6575f67378a09_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128424c649a375d485fb1a6575f67378a09_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2b9050c43df048898caeb6d146185335, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:19,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2b9050c43df048898caeb6d146185335 is 175, key is test_row_0/A:col10/1732758198836/Put/seqid=0 2024-11-28T01:43:19,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742469_1645 (size=30955) 2024-11-28T01:43:19,866 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2b9050c43df048898caeb6d146185335 2024-11-28T01:43:19,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/73b40078404c4b1baad108b983f74ef5 is 50, key is test_row_0/B:col10/1732758198836/Put/seqid=0 2024-11-28T01:43:19,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742470_1646 (size=12001) 2024-11-28T01:43:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:19,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:20,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758259998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758260003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758260004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T01:43:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758260113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758260113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758260113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,148 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#545 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:20,148 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e95e4baeb0534911873d4086c3823b0f is 175, key is test_row_0/A:col10/1732758198805/Put/seqid=0 2024-11-28T01:43:20,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742471_1647 (size=31161) 2024-11-28T01:43:20,280 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/73b40078404c4b1baad108b983f74ef5 2024-11-28T01:43:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/79543bc9f44347528d75d98ceff8dff9 is 50, key is test_row_0/C:col10/1732758198836/Put/seqid=0 2024-11-28T01:43:20,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742472_1648 (size=12001) 2024-11-28T01:43:20,292 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/79543bc9f44347528d75d98ceff8dff9 2024-11-28T01:43:20,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2b9050c43df048898caeb6d146185335 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2b9050c43df048898caeb6d146185335 2024-11-28T01:43:20,299 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2b9050c43df048898caeb6d146185335, entries=150, sequenceid=120, filesize=30.2 K 2024-11-28T01:43:20,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/73b40078404c4b1baad108b983f74ef5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/73b40078404c4b1baad108b983f74ef5 2024-11-28T01:43:20,304 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/73b40078404c4b1baad108b983f74ef5, entries=150, sequenceid=120, filesize=11.7 K 2024-11-28T01:43:20,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/79543bc9f44347528d75d98ceff8dff9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/79543bc9f44347528d75d98ceff8dff9 2024-11-28T01:43:20,308 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/79543bc9f44347528d75d98ceff8dff9, entries=150, sequenceid=120, filesize=11.7 K 2024-11-28T01:43:20,309 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b33d90fd8529f8494aea715fa8bbbcde in 463ms, sequenceid=120, compaction requested=false 2024-11-28T01:43:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-28T01:43:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-28T01:43:20,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-28T01:43:20,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 924 msec 2024-11-28T01:43:20,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 929 msec 2024-11-28T01:43:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:20,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T01:43:20,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:20,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:20,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:20,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:20,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:20,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:20,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112871410947969e46a7aacf89cfa737d1fd_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758200326/Put/seqid=0 2024-11-28T01:43:20,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742473_1649 (size=17284) 2024-11-28T01:43:20,339 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:20,344 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112871410947969e46a7aacf89cfa737d1fd_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112871410947969e46a7aacf89cfa737d1fd_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:20,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/71e782846d41487cad7832bf04bcf78d, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:20,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/71e782846d41487cad7832bf04bcf78d is 175, key is test_row_0/A:col10/1732758200326/Put/seqid=0 2024-11-28T01:43:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742474_1650 (size=48389) 2024-11-28T01:43:20,349 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/71e782846d41487cad7832bf04bcf78d 2024-11-28T01:43:20,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758260350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/cd490bbfc9634de0a42488318dd3c993 is 50, key is test_row_0/B:col10/1732758200326/Put/seqid=0 2024-11-28T01:43:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758260354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758260356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742475_1651 (size=12151) 2024-11-28T01:43:20,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/cd490bbfc9634de0a42488318dd3c993 2024-11-28T01:43:20,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/d6937d280b2e48fb9aa2b2f9d30119b3 is 50, key is test_row_0/C:col10/1732758200326/Put/seqid=0 2024-11-28T01:43:20,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742476_1652 (size=12151) 2024-11-28T01:43:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758260462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758260462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758260472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T01:43:20,523 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:20,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-28T01:43:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T01:43:20,525 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:20,526 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:20,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:20,556 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e95e4baeb0534911873d4086c3823b0f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e95e4baeb0534911873d4086c3823b0f 2024-11-28T01:43:20,560 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into e95e4baeb0534911873d4086c3823b0f(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:20,560 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:20,560 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758199695; duration=0sec 2024-11-28T01:43:20,561 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:20,561 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T01:43:20,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758260672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758260672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-28T01:43:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:20,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758260682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:20,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/d6937d280b2e48fb9aa2b2f9d30119b3 2024-11-28T01:43:20,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/71e782846d41487cad7832bf04bcf78d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/71e782846d41487cad7832bf04bcf78d 2024-11-28T01:43:20,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/71e782846d41487cad7832bf04bcf78d, entries=250, sequenceid=137, filesize=47.3 K 2024-11-28T01:43:20,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/cd490bbfc9634de0a42488318dd3c993 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/cd490bbfc9634de0a42488318dd3c993 2024-11-28T01:43:20,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/cd490bbfc9634de0a42488318dd3c993, entries=150, sequenceid=137, filesize=11.9 K 2024-11-28T01:43:20,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/d6937d280b2e48fb9aa2b2f9d30119b3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d6937d280b2e48fb9aa2b2f9d30119b3 2024-11-28T01:43:20,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d6937d280b2e48fb9aa2b2f9d30119b3, entries=150, sequenceid=137, filesize=11.9 K 2024-11-28T01:43:20,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for b33d90fd8529f8494aea715fa8bbbcde in 470ms, sequenceid=137, compaction requested=true 2024-11-28T01:43:20,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:20,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:20,796 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:20,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:20,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:20,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:20,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:20,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:20,797 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:20,797 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:20,797 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:20,797 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:20,797 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:20,797 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,798 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,798 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f48e3fdcced840cebf74e99068342cdb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/73b40078404c4b1baad108b983f74ef5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/cd490bbfc9634de0a42488318dd3c993] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=35.5 K 2024-11-28T01:43:20,798 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e95e4baeb0534911873d4086c3823b0f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2b9050c43df048898caeb6d146185335, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/71e782846d41487cad7832bf04bcf78d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=107.9 K 2024-11-28T01:43:20,798 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,798 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e95e4baeb0534911873d4086c3823b0f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2b9050c43df048898caeb6d146185335, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/71e782846d41487cad7832bf04bcf78d] 2024-11-28T01:43:20,798 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f48e3fdcced840cebf74e99068342cdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758198484 2024-11-28T01:43:20,798 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e95e4baeb0534911873d4086c3823b0f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758198484 2024-11-28T01:43:20,798 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b40078404c4b1baad108b983f74ef5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758198826 2024-11-28T01:43:20,798 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9050c43df048898caeb6d146185335, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758198826 2024-11-28T01:43:20,799 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting cd490bbfc9634de0a42488318dd3c993, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732758199996 2024-11-28T01:43:20,799 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e782846d41487cad7832bf04bcf78d, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732758199996 2024-11-28T01:43:20,804 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:20,806 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#554 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:20,806 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112890ff73455d534b9eba970d5aee2960b0_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:20,806 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/b7cc0856ddc44eb8beb5305070ccf507 is 50, key is test_row_0/B:col10/1732758200326/Put/seqid=0 2024-11-28T01:43:20,808 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112890ff73455d534b9eba970d5aee2960b0_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:20,808 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112890ff73455d534b9eba970d5aee2960b0_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:20,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742477_1653 (size=12459) 2024-11-28T01:43:20,819 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/b7cc0856ddc44eb8beb5305070ccf507 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/b7cc0856ddc44eb8beb5305070ccf507 2024-11-28T01:43:20,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742478_1654 (size=4469) 2024-11-28T01:43:20,822 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#553 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:20,822 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2ef6a7d1204f461487de65dbdc10c041 is 175, key is test_row_0/A:col10/1732758200326/Put/seqid=0 2024-11-28T01:43:20,827 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into b7cc0856ddc44eb8beb5305070ccf507(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:20,827 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:20,827 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758200796; duration=0sec 2024-11-28T01:43:20,827 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:20,827 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:20,827 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:20,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-28T01:43:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,831 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T01:43:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:20,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:20,833 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:20,833 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:20,833 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:20,834 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c5da3882788e42aca1bd90e48cc1968d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/79543bc9f44347528d75d98ceff8dff9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d6937d280b2e48fb9aa2b2f9d30119b3] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=35.5 K 2024-11-28T01:43:20,834 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c5da3882788e42aca1bd90e48cc1968d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732758198484 2024-11-28T01:43:20,834 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 79543bc9f44347528d75d98ceff8dff9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732758198826 2024-11-28T01:43:20,835 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting d6937d280b2e48fb9aa2b2f9d30119b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732758199996 2024-11-28T01:43:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T01:43:20,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742479_1655 (size=31413) 2024-11-28T01:43:20,859 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2ef6a7d1204f461487de65dbdc10c041 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2ef6a7d1204f461487de65dbdc10c041 2024-11-28T01:43:20,864 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#555 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:20,864 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into 2ef6a7d1204f461487de65dbdc10c041(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:20,864 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:20,864 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758200796; duration=0sec 2024-11-28T01:43:20,864 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:20,864 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:20,864 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/148f178fb8c743079ae3b211433e479c is 50, key is test_row_0/C:col10/1732758200326/Put/seqid=0 2024-11-28T01:43:20,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287f206feb5b284b70bca66a481d5806a8_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758200347/Put/seqid=0 2024-11-28T01:43:20,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742480_1656 (size=12459) 2024-11-28T01:43:20,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742481_1657 (size=12304) 2024-11-28T01:43:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:20,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:21,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758261009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758261010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758261011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758261122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758261122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758261123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T01:43:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:21,276 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/148f178fb8c743079ae3b211433e479c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/148f178fb8c743079ae3b211433e479c 2024-11-28T01:43:21,278 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287f206feb5b284b70bca66a481d5806a8_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287f206feb5b284b70bca66a481d5806a8_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:21,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/ea2b24bda0e847d884e3d69b6ae822db, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:21,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/ea2b24bda0e847d884e3d69b6ae822db is 175, key is test_row_0/A:col10/1732758200347/Put/seqid=0 2024-11-28T01:43:21,285 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into 148f178fb8c743079ae3b211433e479c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:21,285 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:21,285 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758200797; duration=0sec 2024-11-28T01:43:21,286 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:21,286 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:21,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742482_1658 (size=31105) 2024-11-28T01:43:21,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758261333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758261333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758261333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758261582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758261582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,583 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_0' in b33d90fd8529f8494aea715fa8bbbcde of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,584 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_2' in b33d90fd8529f8494aea715fa8bbbcde of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758261642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758261642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758261642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T01:43:21,697 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/ea2b24bda0e847d884e3d69b6ae822db 2024-11-28T01:43:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f3ea3c8c92cd42b4808ab0da1ce93103 is 50, key is test_row_0/B:col10/1732758200347/Put/seqid=0 2024-11-28T01:43:21,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742483_1659 (size=12151) 2024-11-28T01:43:21,716 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f3ea3c8c92cd42b4808ab0da1ce93103 2024-11-28T01:43:21,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/1f96dc9998bf4e13b07095c4dd19db2f is 50, key is test_row_0/C:col10/1732758200347/Put/seqid=0 2024-11-28T01:43:21,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742484_1660 (size=12151) 2024-11-28T01:43:22,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/1f96dc9998bf4e13b07095c4dd19db2f 2024-11-28T01:43:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/ea2b24bda0e847d884e3d69b6ae822db as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/ea2b24bda0e847d884e3d69b6ae822db 2024-11-28T01:43:22,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/ea2b24bda0e847d884e3d69b6ae822db, entries=150, sequenceid=154, filesize=30.4 K 2024-11-28T01:43:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f3ea3c8c92cd42b4808ab0da1ce93103 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3ea3c8c92cd42b4808ab0da1ce93103 2024-11-28T01:43:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,153 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3ea3c8c92cd42b4808ab0da1ce93103, entries=150, sequenceid=154, filesize=11.9 K 2024-11-28T01:43:22,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/1f96dc9998bf4e13b07095c4dd19db2f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/1f96dc9998bf4e13b07095c4dd19db2f 2024-11-28T01:43:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,156 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/1f96dc9998bf4e13b07095c4dd19db2f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-28T01:43:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,157 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b33d90fd8529f8494aea715fa8bbbcde in 1326ms, sequenceid=154, compaction requested=false 2024-11-28T01:43:22,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:22,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:22,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-28T01:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-28T01:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-28T01:43:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-11-28T01:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.6380 sec 2024-11-28T01:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:22,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T01:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:22,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:22,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:22,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:22,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:22,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288a3d50be72804d729359e731192e55d4_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758201009/Put/seqid=0 2024-11-28T01:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742485_1661 (size=12304) 2024-11-28T01:43:22,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288a3d50be72804d729359e731192e55d4_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288a3d50be72804d729359e731192e55d4_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,179 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/de95cd320ba745c7a5c1fcf16b04e0a5, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/de95cd320ba745c7a5c1fcf16b04e0a5 is 175, key is test_row_0/A:col10/1732758201009/Put/seqid=0 2024-11-28T01:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742486_1662 (size=31101) 2024-11-28T01:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:22,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758262199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758262199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758262200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758262313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758262312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758262313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758262522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758262522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758262523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/de95cd320ba745c7a5c1fcf16b04e0a5 2024-11-28T01:43:22,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/72ba9181d532498ab1873aa87035502d is 50, key is test_row_0/B:col10/1732758201009/Put/seqid=0 2024-11-28T01:43:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742487_1663 (size=9757) 2024-11-28T01:43:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T01:43:22,682 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:22,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-28T01:43:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T01:43:22,685 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:22,685 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:22,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T01:43:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758262832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758262832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-28T01:43:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758262832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:22,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:22,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:22,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:22,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:22,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:22,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-28T01:43:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:22,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:22,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/72ba9181d532498ab1873aa87035502d 2024-11-28T01:43:23,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/eb6a124a65884f788f789baa51be0862 is 50, key is test_row_0/C:col10/1732758201009/Put/seqid=0 2024-11-28T01:43:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T01:43:23,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742488_1664 (size=9757) 2024-11-28T01:43:23,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-28T01:43:23,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:23,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:23,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:23,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-28T01:43:23,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:23,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:23,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T01:43:23,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758263343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758263353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:23,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:23,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758263353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:23,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/eb6a124a65884f788f789baa51be0862 2024-11-28T01:43:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/de95cd320ba745c7a5c1fcf16b04e0a5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/de95cd320ba745c7a5c1fcf16b04e0a5 2024-11-28T01:43:23,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/de95cd320ba745c7a5c1fcf16b04e0a5, entries=150, sequenceid=174, filesize=30.4 K 2024-11-28T01:43:23,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/72ba9181d532498ab1873aa87035502d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/72ba9181d532498ab1873aa87035502d 2024-11-28T01:43:23,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/72ba9181d532498ab1873aa87035502d, entries=100, sequenceid=174, filesize=9.5 K 2024-11-28T01:43:23,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/eb6a124a65884f788f789baa51be0862 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/eb6a124a65884f788f789baa51be0862 2024-11-28T01:43:23,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/eb6a124a65884f788f789baa51be0862, entries=100, sequenceid=174, filesize=9.5 K 2024-11-28T01:43:23,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b33d90fd8529f8494aea715fa8bbbcde in 1262ms, sequenceid=174, compaction requested=true 2024-11-28T01:43:23,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:23,425 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:23,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:23,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:23,425 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:23,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:23,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:23,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:23,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:23,426 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:23,426 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:23,427 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,427 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2ef6a7d1204f461487de65dbdc10c041, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/ea2b24bda0e847d884e3d69b6ae822db, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/de95cd320ba745c7a5c1fcf16b04e0a5] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=91.4 K 2024-11-28T01:43:23,427 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:23,427 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:23,427 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,427 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2ef6a7d1204f461487de65dbdc10c041, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/ea2b24bda0e847d884e3d69b6ae822db, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/de95cd320ba745c7a5c1fcf16b04e0a5] 2024-11-28T01:43:23,427 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,427 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/b7cc0856ddc44eb8beb5305070ccf507, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3ea3c8c92cd42b4808ab0da1ce93103, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/72ba9181d532498ab1873aa87035502d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=33.6 K 2024-11-28T01:43:23,427 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting b7cc0856ddc44eb8beb5305070ccf507, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732758199996 2024-11-28T01:43:23,427 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ef6a7d1204f461487de65dbdc10c041, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732758199996 2024-11-28T01:43:23,427 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea2b24bda0e847d884e3d69b6ae822db, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732758200346 2024-11-28T01:43:23,427 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting f3ea3c8c92cd42b4808ab0da1ce93103, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732758200346 2024-11-28T01:43:23,428 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ba9181d532498ab1873aa87035502d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732758201009 2024-11-28T01:43:23,428 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting de95cd320ba745c7a5c1fcf16b04e0a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732758201001 2024-11-28T01:43:23,436 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#562 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:23,436 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/97ce32a21d9647a6986e44b509b6c329 is 50, key is test_row_0/B:col10/1732758201009/Put/seqid=0 2024-11-28T01:43:23,440 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:23,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742489_1665 (size=12561) 2024-11-28T01:43:23,444 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128816068a77e0846d9a006650cfc89cc84_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:23,446 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128816068a77e0846d9a006650cfc89cc84_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:23,446 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128816068a77e0846d9a006650cfc89cc84_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:23,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742490_1666 (size=4469) 2024-11-28T01:43:23,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-28T01:43:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,454 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T01:43:23,454 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#563 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:23,455 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8dcab3658ca94e57ba09a645e29de09d is 175, key is test_row_0/A:col10/1732758201009/Put/seqid=0 2024-11-28T01:43:23,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742491_1667 (size=31622) 2024-11-28T01:43:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283ad78459d7114d5ca0c61717d06de614_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758202192/Put/seqid=0 2024-11-28T01:43:23,464 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8dcab3658ca94e57ba09a645e29de09d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8dcab3658ca94e57ba09a645e29de09d 2024-11-28T01:43:23,469 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into 8dcab3658ca94e57ba09a645e29de09d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:23,469 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:23,469 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758203425; duration=0sec 2024-11-28T01:43:23,469 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:23,469 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:23,469 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:23,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742492_1668 (size=12304) 2024-11-28T01:43:23,470 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:23,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,470 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:23,470 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,471 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/148f178fb8c743079ae3b211433e479c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/1f96dc9998bf4e13b07095c4dd19db2f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/eb6a124a65884f788f789baa51be0862] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=33.6 K 2024-11-28T01:43:23,476 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 148f178fb8c743079ae3b211433e479c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732758199996 2024-11-28T01:43:23,477 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f96dc9998bf4e13b07095c4dd19db2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732758200346 2024-11-28T01:43:23,477 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6a124a65884f788f789baa51be0862, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732758201009 2024-11-28T01:43:23,480 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283ad78459d7114d5ca0c61717d06de614_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283ad78459d7114d5ca0c61717d06de614_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:23,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/3c7b5955c8804d069250c1d76fbd5547, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/3c7b5955c8804d069250c1d76fbd5547 is 175, key is test_row_0/A:col10/1732758202192/Put/seqid=0 2024-11-28T01:43:23,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742493_1669 (size=31105) 2024-11-28T01:43:23,494 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#565 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:23,495 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/39aebc621e6e4892b66e602b2032c468 is 50, key is test_row_0/C:col10/1732758201009/Put/seqid=0 2024-11-28T01:43:23,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742494_1670 (size=12561) 2024-11-28T01:43:23,505 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/39aebc621e6e4892b66e602b2032c468 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/39aebc621e6e4892b66e602b2032c468 2024-11-28T01:43:23,509 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into 39aebc621e6e4892b66e602b2032c468(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:23,510 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:23,510 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758203426; duration=0sec 2024-11-28T01:43:23,510 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:23,510 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T01:43:23,852 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/97ce32a21d9647a6986e44b509b6c329 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/97ce32a21d9647a6986e44b509b6c329 2024-11-28T01:43:23,857 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into 97ce32a21d9647a6986e44b509b6c329(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:23,857 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:23,857 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758203425; duration=0sec 2024-11-28T01:43:23,857 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:23,857 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:23,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/3c7b5955c8804d069250c1d76fbd5547 2024-11-28T01:43:23,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/9cec2434c8d84c9fb6e6d75c5d248ebd is 50, key is test_row_0/B:col10/1732758202192/Put/seqid=0 2024-11-28T01:43:23,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742495_1671 (size=12151) 2024-11-28T01:43:23,914 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/9cec2434c8d84c9fb6e6d75c5d248ebd 2024-11-28T01:43:23,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/6fd117ddc43d4fb3882295b955e0a793 is 50, key is test_row_0/C:col10/1732758202192/Put/seqid=0 2024-11-28T01:43:23,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742496_1672 (size=12151) 2024-11-28T01:43:23,928 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/6fd117ddc43d4fb3882295b955e0a793 2024-11-28T01:43:23,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/3c7b5955c8804d069250c1d76fbd5547 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/3c7b5955c8804d069250c1d76fbd5547 2024-11-28T01:43:23,936 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/3c7b5955c8804d069250c1d76fbd5547, entries=150, sequenceid=188, filesize=30.4 K 2024-11-28T01:43:23,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/9cec2434c8d84c9fb6e6d75c5d248ebd as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/9cec2434c8d84c9fb6e6d75c5d248ebd 2024-11-28T01:43:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/9cec2434c8d84c9fb6e6d75c5d248ebd, entries=150, sequenceid=188, filesize=11.9 K 2024-11-28T01:43:23,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/6fd117ddc43d4fb3882295b955e0a793 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6fd117ddc43d4fb3882295b955e0a793 2024-11-28T01:43:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,944 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6fd117ddc43d4fb3882295b955e0a793, entries=150, sequenceid=188, filesize=11.9 K 2024-11-28T01:43:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,945 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for b33d90fd8529f8494aea715fa8bbbcde in 491ms, sequenceid=188, compaction requested=false 2024-11-28T01:43:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-28T01:43:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-28T01:43:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-28T01:43:23,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2610 sec 2024-11-28T01:43:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.2660 sec 2024-11-28T01:43:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:23,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:43:24,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:24,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:24,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:24,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:24,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:24,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b05cc2dee1a74e00aa10b49b15298a31_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758204395/Put/seqid=0 2024-11-28T01:43:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742497_1673 (size=22268) 2024-11-28T01:43:24,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758204434, max timestamp=1732758204444Num uniques 859; 2024-11-28T01:43:24,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758204434, max timestamp=1732758204444Num uniques 859; 2024-11-28T01:43:24,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:43:24,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:43:24,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732758204434, max timestamp=1732758204444Num uniques 859; 2024-11-28T01:43:24,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T01:43:24,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758264473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758264475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758264476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758264582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758264593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758264593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758264792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758264803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758264803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:24,819 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:24,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b05cc2dee1a74e00aa10b49b15298a31_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b05cc2dee1a74e00aa10b49b15298a31_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:24,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/34721c5213d948d99eba95b15fa0fce3, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:24,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/34721c5213d948d99eba95b15fa0fce3 is 175, key is test_row_0/A:col10/1732758204395/Put/seqid=0 2024-11-28T01:43:24,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742498_1674 (size=65673) 2024-11-28T01:43:24,827 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/34721c5213d948d99eba95b15fa0fce3 2024-11-28T01:43:24,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/0c7ae861bcf540e38b78dee69a6f7e88 is 50, key is test_row_0/B:col10/1732758204395/Put/seqid=0 2024-11-28T01:43:24,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742499_1675 (size=12151) 2024-11-28T01:43:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T01:43:24,842 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:24,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-28T01:43:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T01:43:24,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:24,845 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:24,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T01:43:24,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T01:43:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:24,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:25,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758265102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758265112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758265113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T01:43:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:25,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:25,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T01:43:25,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/0c7ae861bcf540e38b78dee69a6f7e88 2024-11-28T01:43:25,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ece1f86cee844cf1b678dbf63f7d006d is 50, key is test_row_0/C:col10/1732758204395/Put/seqid=0 2024-11-28T01:43:25,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742500_1676 (size=12151) 2024-11-28T01:43:25,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ece1f86cee844cf1b678dbf63f7d006d 2024-11-28T01:43:25,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/34721c5213d948d99eba95b15fa0fce3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/34721c5213d948d99eba95b15fa0fce3 2024-11-28T01:43:25,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/34721c5213d948d99eba95b15fa0fce3, entries=350, sequenceid=201, filesize=64.1 K 2024-11-28T01:43:25,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/0c7ae861bcf540e38b78dee69a6f7e88 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/0c7ae861bcf540e38b78dee69a6f7e88 2024-11-28T01:43:25,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/0c7ae861bcf540e38b78dee69a6f7e88, entries=150, sequenceid=201, filesize=11.9 K 2024-11-28T01:43:25,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ece1f86cee844cf1b678dbf63f7d006d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ece1f86cee844cf1b678dbf63f7d006d 2024-11-28T01:43:25,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ece1f86cee844cf1b678dbf63f7d006d, entries=150, sequenceid=201, filesize=11.9 K 2024-11-28T01:43:25,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b33d90fd8529f8494aea715fa8bbbcde in 875ms, sequenceid=201, compaction requested=true 2024-11-28T01:43:25,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:25,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:25,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:25,271 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:25,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:25,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:25,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:25,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:25,271 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:25,272 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:25,272 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128400 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:25,272 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:25,272 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:25,272 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:25,272 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:25,272 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8dcab3658ca94e57ba09a645e29de09d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/3c7b5955c8804d069250c1d76fbd5547, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/34721c5213d948d99eba95b15fa0fce3] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=125.4 K 2024-11-28T01:43:25,272 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/97ce32a21d9647a6986e44b509b6c329, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/9cec2434c8d84c9fb6e6d75c5d248ebd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/0c7ae861bcf540e38b78dee69a6f7e88] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.0 K 2024-11-28T01:43:25,272 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:25,272 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8dcab3658ca94e57ba09a645e29de09d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/3c7b5955c8804d069250c1d76fbd5547, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/34721c5213d948d99eba95b15fa0fce3] 2024-11-28T01:43:25,273 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 97ce32a21d9647a6986e44b509b6c329, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732758200354 2024-11-28T01:43:25,273 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dcab3658ca94e57ba09a645e29de09d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732758200354 2024-11-28T01:43:25,274 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cec2434c8d84c9fb6e6d75c5d248ebd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732758202190 2024-11-28T01:43:25,274 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c7b5955c8804d069250c1d76fbd5547, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732758202190 2024-11-28T01:43:25,274 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7ae861bcf540e38b78dee69a6f7e88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732758204371 2024-11-28T01:43:25,274 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34721c5213d948d99eba95b15fa0fce3, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732758204371 2024-11-28T01:43:25,280 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:25,282 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#572 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:25,282 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/efac638cdabd4c66a3f142e8f5f3a532 is 50, key is test_row_0/B:col10/1732758204395/Put/seqid=0 2024-11-28T01:43:25,284 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128de74bdd998b04ca48407303609129c1b_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:25,286 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128de74bdd998b04ca48407303609129c1b_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:25,286 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128de74bdd998b04ca48407303609129c1b_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:25,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742501_1677 (size=12663) 2024-11-28T01:43:25,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742502_1678 (size=4469) 2024-11-28T01:43:25,296 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#571 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:25,297 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/c75b0e396cca427995e5005052dab369 is 175, key is test_row_0/A:col10/1732758204395/Put/seqid=0 2024-11-28T01:43:25,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742503_1679 (size=31617) 2024-11-28T01:43:25,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T01:43:25,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:25,304 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-28T01:43:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283fbd614b5ea24e579b8aceb92e000651_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758204468/Put/seqid=0 2024-11-28T01:43:25,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742504_1680 (size=12304) 2024-11-28T01:43:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T01:43:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:25,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:25,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758265629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758265630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758265630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758265632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732758265632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,639 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9df3f25f5867,42371,1732758050017 for 'test_row_0' in b33d90fd8529f8494aea715fa8bbbcde of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T01:43:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,698 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/efac638cdabd4c66a3f142e8f5f3a532 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/efac638cdabd4c66a3f142e8f5f3a532 2024-11-28T01:43:25,702 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into efac638cdabd4c66a3f142e8f5f3a532(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:25,702 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:25,702 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758205271; duration=0sec 2024-11-28T01:43:25,703 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:25,703 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:25,703 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:25,704 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:25,704 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:25,705 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:25,705 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/39aebc621e6e4892b66e602b2032c468, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6fd117ddc43d4fb3882295b955e0a793, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ece1f86cee844cf1b678dbf63f7d006d] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.0 K 2024-11-28T01:43:25,706 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/c75b0e396cca427995e5005052dab369 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/c75b0e396cca427995e5005052dab369 2024-11-28T01:43:25,706 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 39aebc621e6e4892b66e602b2032c468, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732758200354 2024-11-28T01:43:25,706 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd117ddc43d4fb3882295b955e0a793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732758202190 2024-11-28T01:43:25,706 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ece1f86cee844cf1b678dbf63f7d006d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732758204371 2024-11-28T01:43:25,710 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into c75b0e396cca427995e5005052dab369(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:25,710 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:25,710 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758205271; duration=0sec 2024-11-28T01:43:25,710 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:25,710 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:25,712 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#574 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:25,713 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ce6127d115b04fe6b170656bf1953e17 is 50, key is test_row_0/C:col10/1732758204395/Put/seqid=0 2024-11-28T01:43:25,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:25,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742505_1681 (size=12663) 2024-11-28T01:43:25,719 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283fbd614b5ea24e579b8aceb92e000651_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283fbd614b5ea24e579b8aceb92e000651_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:25,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/f37841d1174a42e1aef4197112f1cd13, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:25,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/f37841d1174a42e1aef4197112f1cd13 is 175, key is test_row_0/A:col10/1732758204468/Put/seqid=0 2024-11-28T01:43:25,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742506_1682 (size=31105) 2024-11-28T01:43:25,724 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ce6127d115b04fe6b170656bf1953e17 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ce6127d115b04fe6b170656bf1953e17 2024-11-28T01:43:25,731 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into ce6127d115b04fe6b170656bf1953e17(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:25,731 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:25,731 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758205271; duration=0sec 2024-11-28T01:43:25,731 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:25,731 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758265743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758265743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758265743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758265743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758265953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758265953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758265953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758265953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T01:43:26,125 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/f37841d1174a42e1aef4197112f1cd13 2024-11-28T01:43:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/aeb55721c7de4b53ab1a87c5d59ec09b is 50, key is test_row_0/B:col10/1732758204468/Put/seqid=0 2024-11-28T01:43:26,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742507_1683 (size=12151) 2024-11-28T01:43:26,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758266263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758266263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758266263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758266263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,537 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/aeb55721c7de4b53ab1a87c5d59ec09b 2024-11-28T01:43:26,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/f6a5f442096146f8b2030b9064f9ab31 is 50, key is test_row_0/C:col10/1732758204468/Put/seqid=0 2024-11-28T01:43:26,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742508_1684 (size=12151) 2024-11-28T01:43:26,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758266773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758266773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758266773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758266773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:26,949 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/f6a5f442096146f8b2030b9064f9ab31 2024-11-28T01:43:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/f37841d1174a42e1aef4197112f1cd13 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f37841d1174a42e1aef4197112f1cd13 2024-11-28T01:43:26,956 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f37841d1174a42e1aef4197112f1cd13, entries=150, sequenceid=236, filesize=30.4 K 2024-11-28T01:43:26,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/aeb55721c7de4b53ab1a87c5d59ec09b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aeb55721c7de4b53ab1a87c5d59ec09b 2024-11-28T01:43:26,960 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aeb55721c7de4b53ab1a87c5d59ec09b, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T01:43:26,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/f6a5f442096146f8b2030b9064f9ab31 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f6a5f442096146f8b2030b9064f9ab31 2024-11-28T01:43:26,965 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f6a5f442096146f8b2030b9064f9ab31, entries=150, sequenceid=236, filesize=11.9 K 2024-11-28T01:43:26,965 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for b33d90fd8529f8494aea715fa8bbbcde in 1661ms, sequenceid=236, compaction requested=false 2024-11-28T01:43:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:26,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-28T01:43:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-28T01:43:26,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-28T01:43:26,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1220 sec 2024-11-28T01:43:26,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.1270 sec 2024-11-28T01:43:27,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T01:43:27,003 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:27,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-28T01:43:27,005 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:27,006 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:27,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T01:43:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T01:43:27,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T01:43:27,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:27,159 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T01:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:27,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112817c56812971742f6a5a26821a663e597_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758205630/Put/seqid=0 2024-11-28T01:43:27,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742509_1685 (size=12304) 2024-11-28T01:43:27,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:27,173 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112817c56812971742f6a5a26821a663e597_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112817c56812971742f6a5a26821a663e597_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:27,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e73a145c783341db97f183cbeb4b0e26, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:27,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e73a145c783341db97f183cbeb4b0e26 is 175, key is test_row_0/A:col10/1732758205630/Put/seqid=0 2024-11-28T01:43:27,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742510_1686 (size=31105) 2024-11-28T01:43:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T01:43:27,578 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e73a145c783341db97f183cbeb4b0e26 2024-11-28T01:43:27,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/ba8409cb828b41a298ca0fbf842f4e35 is 50, key is test_row_0/B:col10/1732758205630/Put/seqid=0 2024-11-28T01:43:27,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742511_1687 (size=12151) 2024-11-28T01:43:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T01:43:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:27,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:27,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758267813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758207805, max timestamp=1732758207818Num uniques 859; 2024-11-28T01:43:27,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758207805, max timestamp=1732758207818Num uniques 859; 2024-11-28T01:43:27,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:43:27,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:43:27,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732758207805, max timestamp=1732758207818Num uniques 859; 2024-11-28T01:43:27,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T01:43:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:27,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758267858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758267858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758267859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:27,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758267933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:27,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758267972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758267972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:27,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758267972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/ba8409cb828b41a298ca0fbf842f4e35 2024-11-28T01:43:28,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/e720314297414d1f92cb976ede67d3f7 is 50, key is test_row_0/C:col10/1732758205630/Put/seqid=0 2024-11-28T01:43:28,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742512_1688 (size=12151) 2024-11-28T01:43:28,039 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/e720314297414d1f92cb976ede67d3f7 2024-11-28T01:43:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e73a145c783341db97f183cbeb4b0e26 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e73a145c783341db97f183cbeb4b0e26 2024-11-28T01:43:28,047 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e73a145c783341db97f183cbeb4b0e26, entries=150, sequenceid=249, filesize=30.4 K 2024-11-28T01:43:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/ba8409cb828b41a298ca0fbf842f4e35 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/ba8409cb828b41a298ca0fbf842f4e35 2024-11-28T01:43:28,050 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/ba8409cb828b41a298ca0fbf842f4e35, entries=150, sequenceid=249, filesize=11.9 K 2024-11-28T01:43:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/e720314297414d1f92cb976ede67d3f7 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/e720314297414d1f92cb976ede67d3f7 2024-11-28T01:43:28,054 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/e720314297414d1f92cb976ede67d3f7, entries=150, sequenceid=249, filesize=11.9 K 2024-11-28T01:43:28,055 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for b33d90fd8529f8494aea715fa8bbbcde in 896ms, sequenceid=249, compaction requested=true 2024-11-28T01:43:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-28T01:43:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-28T01:43:28,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-28T01:43:28,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0500 sec 2024-11-28T01:43:28,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.0550 sec 2024-11-28T01:43:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T01:43:28,143 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:28,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-28T01:43:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:28,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-28T01:43:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T01:43:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:43:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:43:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:28,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T01:43:28,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:28,150 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:28,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:28,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a1388ac4a95e47e5b24e1552da926882_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758207856/Put/seqid=0 2024-11-28T01:43:28,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742513_1689 (size=14994) 2024-11-28T01:43:28,157 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:28,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a1388ac4a95e47e5b24e1552da926882_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a1388ac4a95e47e5b24e1552da926882_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:28,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/7d241cd3599047c3907ee7a7c26370a6, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:28,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/7d241cd3599047c3907ee7a7c26370a6 is 175, key is test_row_0/A:col10/1732758207856/Put/seqid=0 2024-11-28T01:43:28,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742514_1690 (size=39949) 2024-11-28T01:43:28,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/7d241cd3599047c3907ee7a7c26370a6 2024-11-28T01:43:28,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758268183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758268183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758268183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758268189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/18bb215416c44cd2a02b469a02ca6582 is 50, key is test_row_0/B:col10/1732758207856/Put/seqid=0 2024-11-28T01:43:28,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742515_1691 (size=12301) 2024-11-28T01:43:28,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/18bb215416c44cd2a02b469a02ca6582 2024-11-28T01:43:28,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/31ee90b444cd4be38af8f4daf6f1a146 is 50, key is test_row_0/C:col10/1732758207856/Put/seqid=0 2024-11-28T01:43:28,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742516_1692 (size=12301) 2024-11-28T01:43:28,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/31ee90b444cd4be38af8f4daf6f1a146 2024-11-28T01:43:28,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/7d241cd3599047c3907ee7a7c26370a6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/7d241cd3599047c3907ee7a7c26370a6 2024-11-28T01:43:28,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/7d241cd3599047c3907ee7a7c26370a6, entries=200, sequenceid=287, filesize=39.0 K 2024-11-28T01:43:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/18bb215416c44cd2a02b469a02ca6582 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/18bb215416c44cd2a02b469a02ca6582 2024-11-28T01:43:28,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/18bb215416c44cd2a02b469a02ca6582, entries=150, sequenceid=287, filesize=12.0 K 2024-11-28T01:43:28,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/31ee90b444cd4be38af8f4daf6f1a146 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/31ee90b444cd4be38af8f4daf6f1a146 2024-11-28T01:43:28,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/31ee90b444cd4be38af8f4daf6f1a146, entries=150, sequenceid=287, filesize=12.0 K 2024-11-28T01:43:28,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for b33d90fd8529f8494aea715fa8bbbcde in 97ms, sequenceid=287, compaction requested=true 2024-11-28T01:43:28,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:28,243 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:43:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:28,243 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:43:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:28,245 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133776 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:43:28,245 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:28,245 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,245 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/c75b0e396cca427995e5005052dab369, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f37841d1174a42e1aef4197112f1cd13, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e73a145c783341db97f183cbeb4b0e26, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/7d241cd3599047c3907ee7a7c26370a6] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=130.6 K 2024-11-28T01:43:28,245 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,245 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/c75b0e396cca427995e5005052dab369, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f37841d1174a42e1aef4197112f1cd13, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e73a145c783341db97f183cbeb4b0e26, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/7d241cd3599047c3907ee7a7c26370a6] 2024-11-28T01:43:28,245 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:43:28,245 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:28,245 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,245 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/efac638cdabd4c66a3f142e8f5f3a532, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aeb55721c7de4b53ab1a87c5d59ec09b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/ba8409cb828b41a298ca0fbf842f4e35, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/18bb215416c44cd2a02b469a02ca6582] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=48.1 K 2024-11-28T01:43:28,246 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting efac638cdabd4c66a3f142e8f5f3a532, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732758204371 2024-11-28T01:43:28,246 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c75b0e396cca427995e5005052dab369, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732758204371 2024-11-28T01:43:28,247 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb55721c7de4b53ab1a87c5d59ec09b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758204468 2024-11-28T01:43:28,248 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f37841d1174a42e1aef4197112f1cd13, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758204468 2024-11-28T01:43:28,248 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ba8409cb828b41a298ca0fbf842f4e35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732758205626 2024-11-28T01:43:28,249 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 18bb215416c44cd2a02b469a02ca6582, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732758207856 2024-11-28T01:43:28,249 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e73a145c783341db97f183cbeb4b0e26, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732758205626 2024-11-28T01:43:28,250 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d241cd3599047c3907ee7a7c26370a6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732758207847 2024-11-28T01:43:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T01:43:28,271 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#583 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:28,272 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/51c800ba4ed94675ad8d3013227aaff0 is 50, key is test_row_0/B:col10/1732758207856/Put/seqid=0 2024-11-28T01:43:28,276 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:28,284 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411288569349c27fb4aafae8921ed8c95856d_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:28,287 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411288569349c27fb4aafae8921ed8c95856d_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:28,287 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288569349c27fb4aafae8921ed8c95856d_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:28,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T01:43:28,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,303 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T01:43:28,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:28,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:28,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:28,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742517_1693 (size=12949) 2024-11-28T01:43:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:28,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:28,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128da74f51d62384f1794c738091f2c5f36_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758208170/Put/seqid=0 2024-11-28T01:43:28,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742518_1694 (size=4469) 2024-11-28T01:43:28,324 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#584 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:28,325 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/950d8af49df14e17b790a84f4157c125 is 175, key is test_row_0/A:col10/1732758207856/Put/seqid=0 2024-11-28T01:43:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742519_1695 (size=12454) 2024-11-28T01:43:28,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:28,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128da74f51d62384f1794c738091f2c5f36_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128da74f51d62384f1794c738091f2c5f36_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:28,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8b90e0140a98453d8ff1ef842203d0f9, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:28,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8b90e0140a98453d8ff1ef842203d0f9 is 175, key is test_row_0/A:col10/1732758208170/Put/seqid=0 2024-11-28T01:43:28,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742520_1696 (size=31903) 2024-11-28T01:43:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742521_1697 (size=31255) 2024-11-28T01:43:28,363 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8b90e0140a98453d8ff1ef842203d0f9 2024-11-28T01:43:28,368 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/950d8af49df14e17b790a84f4157c125 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/950d8af49df14e17b790a84f4157c125 2024-11-28T01:43:28,375 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into 950d8af49df14e17b790a84f4157c125(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:28,375 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:28,376 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=12, startTime=1732758208242; duration=0sec 2024-11-28T01:43:28,376 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:28,376 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:28,376 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T01:43:28,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/50fc4dea1a85434b90c00c30475412f8 is 50, key is test_row_0/B:col10/1732758208170/Put/seqid=0 2024-11-28T01:43:28,378 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T01:43:28,378 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:28,378 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,379 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ce6127d115b04fe6b170656bf1953e17, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f6a5f442096146f8b2030b9064f9ab31, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/e720314297414d1f92cb976ede67d3f7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/31ee90b444cd4be38af8f4daf6f1a146] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=48.1 K 2024-11-28T01:43:28,379 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6127d115b04fe6b170656bf1953e17, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732758204371 2024-11-28T01:43:28,380 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6a5f442096146f8b2030b9064f9ab31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732758204468 2024-11-28T01:43:28,380 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e720314297414d1f92cb976ede67d3f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732758205626 2024-11-28T01:43:28,381 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ee90b444cd4be38af8f4daf6f1a146, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732758207856 2024-11-28T01:43:28,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742522_1698 (size=12301) 2024-11-28T01:43:28,406 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#587 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:28,407 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/05ca0be05741443aba5d19f1da81075e is 50, key is test_row_0/C:col10/1732758207856/Put/seqid=0 2024-11-28T01:43:28,410 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/50fc4dea1a85434b90c00c30475412f8 2024-11-28T01:43:28,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758208379, max timestamp=1732758208423Num uniques 859; 2024-11-28T01:43:28,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:43:28,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732758208379, max timestamp=1732758208423Num uniques 859; 2024-11-28T01:43:28,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9df3f25f5867:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T01:43:28,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ab2a714ffc304f19adce4e7f7f22849b is 50, key is test_row_0/C:col10/1732758208170/Put/seqid=0 2024-11-28T01:43:28,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742523_1699 (size=12949) 2024-11-28T01:43:28,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742524_1700 (size=12301) 2024-11-28T01:43:28,451 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ab2a714ffc304f19adce4e7f7f22849b 2024-11-28T01:43:28,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8b90e0140a98453d8ff1ef842203d0f9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8b90e0140a98453d8ff1ef842203d0f9 2024-11-28T01:43:28,462 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8b90e0140a98453d8ff1ef842203d0f9, entries=150, sequenceid=295, filesize=30.5 K 2024-11-28T01:43:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T01:43:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/50fc4dea1a85434b90c00c30475412f8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/50fc4dea1a85434b90c00c30475412f8 2024-11-28T01:43:28,477 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/50fc4dea1a85434b90c00c30475412f8, entries=150, sequenceid=295, filesize=12.0 K 2024-11-28T01:43:28,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/ab2a714ffc304f19adce4e7f7f22849b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ab2a714ffc304f19adce4e7f7f22849b 2024-11-28T01:43:28,481 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ab2a714ffc304f19adce4e7f7f22849b, entries=150, sequenceid=295, filesize=12.0 K 2024-11-28T01:43:28,482 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=105.11 KB/107630 for b33d90fd8529f8494aea715fa8bbbcde in 179ms, sequenceid=295, compaction requested=false 2024-11-28T01:43:28,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:28,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-28T01:43:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-28T01:43:28,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-28T01:43:28,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 333 msec 2024-11-28T01:43:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:28,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=111.82 KB heapSize=323.55 KB 2024-11-28T01:43:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:28,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T01:43:28,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 343 msec 2024-11-28T01:43:28,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281e6e977fb9714ba7865d7d3fca0b39ba_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758208432/Put/seqid=0 2024-11-28T01:43:28,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742525_1701 (size=14994) 2024-11-28T01:43:28,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758268520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758268521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758268522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758268522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758268632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758268632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758268642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758268643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,713 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/51c800ba4ed94675ad8d3013227aaff0 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/51c800ba4ed94675ad8d3013227aaff0 2024-11-28T01:43:28,717 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into 51c800ba4ed94675ad8d3013227aaff0(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:28,717 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:28,717 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=12, startTime=1732758208243; duration=0sec 2024-11-28T01:43:28,717 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:28,717 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T01:43:28,773 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:28,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-28T01:43:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T01:43:28,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:28,776 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:28,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:28,840 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/05ca0be05741443aba5d19f1da81075e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/05ca0be05741443aba5d19f1da81075e 2024-11-28T01:43:28,844 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into 05ca0be05741443aba5d19f1da81075e(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:28,845 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:28,845 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=12, startTime=1732758208243; duration=0sec 2024-11-28T01:43:28,845 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:28,845 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758268842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758268842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758268853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:28,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758268853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T01:43:28,914 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:28,917 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281e6e977fb9714ba7865d7d3fca0b39ba_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281e6e977fb9714ba7865d7d3fca0b39ba_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:28,918 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/1134b139a07e4fec8fba14b6c94fcfae, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:28,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/1134b139a07e4fec8fba14b6c94fcfae is 175, key is test_row_0/A:col10/1732758208432/Put/seqid=0 2024-11-28T01:43:28,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742526_1702 (size=39949) 2024-11-28T01:43:28,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:28,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:28,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T01:43:29,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758269152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758269163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758269163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758269172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:29,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/1134b139a07e4fec8fba14b6c94fcfae 2024-11-28T01:43:29,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/6e0785e8b7084339b8dc33474a47a258 is 50, key is test_row_0/B:col10/1732758208432/Put/seqid=0 2024-11-28T01:43:29,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742527_1703 (size=12301) 2024-11-28T01:43:29,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T01:43:29,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758269663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758269673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758269673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758269682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:29,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/6e0785e8b7084339b8dc33474a47a258 2024-11-28T01:43:29,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/c98999c3e048487d9ebd9adec875ed51 is 50, key is test_row_0/C:col10/1732758208432/Put/seqid=0 2024-11-28T01:43:29,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742528_1704 (size=12301) 2024-11-28T01:43:29,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:29,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:29,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:29,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T01:43:30,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:30,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:30,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:30,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:30,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/c98999c3e048487d9ebd9adec875ed51 2024-11-28T01:43:30,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/1134b139a07e4fec8fba14b6c94fcfae as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/1134b139a07e4fec8fba14b6c94fcfae 2024-11-28T01:43:30,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:30,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:30,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/1134b139a07e4fec8fba14b6c94fcfae, entries=200, sequenceid=325, filesize=39.0 K 2024-11-28T01:43:30,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/6e0785e8b7084339b8dc33474a47a258 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/6e0785e8b7084339b8dc33474a47a258 2024-11-28T01:43:30,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/6e0785e8b7084339b8dc33474a47a258, entries=150, sequenceid=325, filesize=12.0 K 2024-11-28T01:43:30,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/c98999c3e048487d9ebd9adec875ed51 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c98999c3e048487d9ebd9adec875ed51 2024-11-28T01:43:30,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c98999c3e048487d9ebd9adec875ed51, entries=150, sequenceid=325, filesize=12.0 K 2024-11-28T01:43:30,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~323.50 KB/331264, currentSize=67.09 KB/68700 for b33d90fd8529f8494aea715fa8bbbcde in 1685ms, sequenceid=325, compaction requested=true 2024-11-28T01:43:30,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:30,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:30,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:30,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:30,173 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:30,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:30,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:30,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:30,173 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:30,174 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:30,174 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:30,174 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:30,174 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:30,174 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,175 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,175 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/51c800ba4ed94675ad8d3013227aaff0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/50fc4dea1a85434b90c00c30475412f8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/6e0785e8b7084339b8dc33474a47a258] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.7 K 2024-11-28T01:43:30,175 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/950d8af49df14e17b790a84f4157c125, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8b90e0140a98453d8ff1ef842203d0f9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/1134b139a07e4fec8fba14b6c94fcfae] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=100.7 K 2024-11-28T01:43:30,175 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,175 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/950d8af49df14e17b790a84f4157c125, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8b90e0140a98453d8ff1ef842203d0f9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/1134b139a07e4fec8fba14b6c94fcfae] 2024-11-28T01:43:30,175 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c800ba4ed94675ad8d3013227aaff0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732758207856 2024-11-28T01:43:30,175 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 950d8af49df14e17b790a84f4157c125, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732758207856 2024-11-28T01:43:30,175 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 50fc4dea1a85434b90c00c30475412f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732758208163 2024-11-28T01:43:30,176 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b90e0140a98453d8ff1ef842203d0f9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732758208163 2024-11-28T01:43:30,176 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0785e8b7084339b8dc33474a47a258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732758208432 2024-11-28T01:43:30,176 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1134b139a07e4fec8fba14b6c94fcfae, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732758208432 2024-11-28T01:43:30,203 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#592 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:30,204 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/78233b6878a1444e8b3fbddb8e154168 is 50, key is test_row_0/B:col10/1732758208432/Put/seqid=0 2024-11-28T01:43:30,206 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:30,211 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112855729252b69f4e1f8482ce4e4bcea9a1_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:30,214 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112855729252b69f4e1f8482ce4e4bcea9a1_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:30,214 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112855729252b69f4e1f8482ce4e4bcea9a1_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:30,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742529_1705 (size=13051) 2024-11-28T01:43:30,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742530_1706 (size=4469) 2024-11-28T01:43:30,227 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#593 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:30,228 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/048a0653a4614d05908b58027d8ade14 is 175, key is test_row_0/A:col10/1732758208432/Put/seqid=0 2024-11-28T01:43:30,230 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/78233b6878a1444e8b3fbddb8e154168 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/78233b6878a1444e8b3fbddb8e154168 2024-11-28T01:43:30,235 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into 78233b6878a1444e8b3fbddb8e154168(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:30,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742531_1707 (size=32005) 2024-11-28T01:43:30,235 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:30,235 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758210173; duration=0sec 2024-11-28T01:43:30,236 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:30,236 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:30,236 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:30,238 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:30,238 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:30,238 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,238 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/05ca0be05741443aba5d19f1da81075e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ab2a714ffc304f19adce4e7f7f22849b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c98999c3e048487d9ebd9adec875ed51] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.7 K 2024-11-28T01:43:30,238 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ca0be05741443aba5d19f1da81075e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732758207856 2024-11-28T01:43:30,240 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting ab2a714ffc304f19adce4e7f7f22849b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732758208163 2024-11-28T01:43:30,240 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting c98999c3e048487d9ebd9adec875ed51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732758208432 2024-11-28T01:43:30,243 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/048a0653a4614d05908b58027d8ade14 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/048a0653a4614d05908b58027d8ade14 2024-11-28T01:43:30,249 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into 048a0653a4614d05908b58027d8ade14(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:30,249 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:30,250 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758210172; duration=0sec 2024-11-28T01:43:30,250 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:30,250 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:30,253 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#594 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:30,254 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/0179181e703949f1b2d3060d1b8d996a is 50, key is test_row_0/C:col10/1732758208432/Put/seqid=0 2024-11-28T01:43:30,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742532_1708 (size=13051) 2024-11-28T01:43:30,263 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/0179181e703949f1b2d3060d1b8d996a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/0179181e703949f1b2d3060d1b8d996a 2024-11-28T01:43:30,268 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into 0179181e703949f1b2d3060d1b8d996a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:30,268 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:30,268 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758210173; duration=0sec 2024-11-28T01:43:30,268 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:30,268 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:30,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T01:43:30,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,311 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:43:30,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:30,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:30,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:30,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:30,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:30,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:30,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b1b58f6b68d64e3aad4ac6b6248f5fa9_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758208519/Put/seqid=0 2024-11-28T01:43:30,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742533_1709 (size=12454) 2024-11-28T01:43:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:30,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:30,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758270705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758270707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758270708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758270710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:30,727 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b1b58f6b68d64e3aad4ac6b6248f5fa9_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b1b58f6b68d64e3aad4ac6b6248f5fa9_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:30,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/cf5c2a9fb59842daab2a5af8e1d1a966, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/cf5c2a9fb59842daab2a5af8e1d1a966 is 175, key is test_row_0/A:col10/1732758208519/Put/seqid=0 2024-11-28T01:43:30,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742534_1710 (size=31255) 2024-11-28T01:43:30,741 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/cf5c2a9fb59842daab2a5af8e1d1a966 2024-11-28T01:43:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/64e77478affe4fa590889c26dde3e544 is 50, key is test_row_0/B:col10/1732758208519/Put/seqid=0 2024-11-28T01:43:30,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742535_1711 (size=12301) 2024-11-28T01:43:30,762 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/64e77478affe4fa590889c26dde3e544 2024-11-28T01:43:30,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/a7d66a0be3954215874e4e0af9fa7a1e is 50, key is test_row_0/C:col10/1732758208519/Put/seqid=0 2024-11-28T01:43:30,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742536_1712 (size=12301) 2024-11-28T01:43:30,782 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/a7d66a0be3954215874e4e0af9fa7a1e 2024-11-28T01:43:30,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/cf5c2a9fb59842daab2a5af8e1d1a966 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/cf5c2a9fb59842daab2a5af8e1d1a966 2024-11-28T01:43:30,791 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/cf5c2a9fb59842daab2a5af8e1d1a966, entries=150, sequenceid=343, filesize=30.5 K 2024-11-28T01:43:30,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/64e77478affe4fa590889c26dde3e544 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/64e77478affe4fa590889c26dde3e544 2024-11-28T01:43:30,796 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/64e77478affe4fa590889c26dde3e544, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T01:43:30,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/a7d66a0be3954215874e4e0af9fa7a1e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/a7d66a0be3954215874e4e0af9fa7a1e 2024-11-28T01:43:30,801 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/a7d66a0be3954215874e4e0af9fa7a1e, entries=150, sequenceid=343, filesize=12.0 K 2024-11-28T01:43:30,801 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b33d90fd8529f8494aea715fa8bbbcde in 490ms, sequenceid=343, compaction requested=false 2024-11-28T01:43:30,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:30,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:30,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-28T01:43:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-28T01:43:30,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-28T01:43:30,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-28T01:43:30,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.0320 sec 2024-11-28T01:43:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:30,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:43:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:30,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128272b741811d640b880b81b1daa118186_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758210705/Put/seqid=0 2024-11-28T01:43:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742537_1713 (size=14994) 2024-11-28T01:43:30,833 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:30,839 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128272b741811d640b880b81b1daa118186_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128272b741811d640b880b81b1daa118186_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:30,840 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/5f6a45a197924ed7868b7e019ee84cd8, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:30,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/5f6a45a197924ed7868b7e019ee84cd8 is 175, key is test_row_0/A:col10/1732758210705/Put/seqid=0 2024-11-28T01:43:30,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758270836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742538_1714 (size=39949) 2024-11-28T01:43:30,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758270840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758270840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758270841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T01:43:30,923 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:30,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-28T01:43:30,925 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T01:43:30,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:30,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758270953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758270953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758270953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758270953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T01:43:31,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T01:43:31,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758271162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758271162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758271162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758271163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T01:43:31,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T01:43:31,245 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/5f6a45a197924ed7868b7e019ee84cd8 2024-11-28T01:43:31,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/8e3173980b814de0af21015a5b1d7b41 is 50, key is test_row_0/B:col10/1732758210705/Put/seqid=0 2024-11-28T01:43:31,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742539_1715 (size=12301) 2024-11-28T01:43:31,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/8e3173980b814de0af21015a5b1d7b41 2024-11-28T01:43:31,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/dff19389d1184a49a666b0b6e6f925a3 is 50, key is test_row_0/C:col10/1732758210705/Put/seqid=0 2024-11-28T01:43:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742540_1716 (size=12301) 2024-11-28T01:43:31,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T01:43:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758271472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758271472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758271472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758271472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:31,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T01:43:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T01:43:31,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/dff19389d1184a49a666b0b6e6f925a3 2024-11-28T01:43:31,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/5f6a45a197924ed7868b7e019ee84cd8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/5f6a45a197924ed7868b7e019ee84cd8 2024-11-28T01:43:31,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/5f6a45a197924ed7868b7e019ee84cd8, entries=200, sequenceid=363, filesize=39.0 K 2024-11-28T01:43:31,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/8e3173980b814de0af21015a5b1d7b41 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/8e3173980b814de0af21015a5b1d7b41 2024-11-28T01:43:31,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/8e3173980b814de0af21015a5b1d7b41, entries=150, sequenceid=363, filesize=12.0 K 2024-11-28T01:43:31,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T01:43:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:43:31,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/dff19389d1184a49a666b0b6e6f925a3 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/dff19389d1184a49a666b0b6e6f925a3 2024-11-28T01:43:31,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/dff19389d1184a49a666b0b6e6f925a3, entries=150, sequenceid=363, filesize=12.0 K 2024-11-28T01:43:31,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for b33d90fd8529f8494aea715fa8bbbcde in 880ms, sequenceid=363, compaction requested=true 2024-11-28T01:43:31,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:31,696 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:31,698 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:31,698 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:31,698 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,698 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/048a0653a4614d05908b58027d8ade14, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/cf5c2a9fb59842daab2a5af8e1d1a966, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/5f6a45a197924ed7868b7e019ee84cd8] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=100.8 K 2024-11-28T01:43:31,698 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,698 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/048a0653a4614d05908b58027d8ade14, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/cf5c2a9fb59842daab2a5af8e1d1a966, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/5f6a45a197924ed7868b7e019ee84cd8] 2024-11-28T01:43:31,698 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048a0653a4614d05908b58027d8ade14, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732758208432 2024-11-28T01:43:31,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:31,698 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5c2a9fb59842daab2a5af8e1d1a966, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732758208519 2024-11-28T01:43:31,699 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6a45a197924ed7868b7e019ee84cd8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732758210698 2024-11-28T01:43:31,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:31,703 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:31,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:31,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:31,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:31,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:31,704 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:31,705 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:31,705 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:31,705 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,705 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/78233b6878a1444e8b3fbddb8e154168, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/64e77478affe4fa590889c26dde3e544, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/8e3173980b814de0af21015a5b1d7b41] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.8 K 2024-11-28T01:43:31,706 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 78233b6878a1444e8b3fbddb8e154168, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732758208432 2024-11-28T01:43:31,706 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e77478affe4fa590889c26dde3e544, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732758208519 2024-11-28T01:43:31,707 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3173980b814de0af21015a5b1d7b41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732758210705 2024-11-28T01:43:31,716 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128b8c267e0ea4a4b718c5b7043914877ef_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:31,718 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#602 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:31,718 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/aba4107402b5453aaf9b7ae8e5c9c221 is 50, key is test_row_0/B:col10/1732758210705/Put/seqid=0 2024-11-28T01:43:31,720 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128b8c267e0ea4a4b718c5b7043914877ef_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:31,720 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b8c267e0ea4a4b718c5b7043914877ef_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:31,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742542_1718 (size=4469) 2024-11-28T01:43:31,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742541_1717 (size=13153) 2024-11-28T01:43:31,741 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/aba4107402b5453aaf9b7ae8e5c9c221 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aba4107402b5453aaf9b7ae8e5c9c221 2024-11-28T01:43:31,745 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into aba4107402b5453aaf9b7ae8e5c9c221(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:31,745 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:31,745 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758211703; duration=0sec 2024-11-28T01:43:31,745 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:31,745 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:31,745 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:31,753 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:31,753 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:31,753 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,753 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/0179181e703949f1b2d3060d1b8d996a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/a7d66a0be3954215874e4e0af9fa7a1e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/dff19389d1184a49a666b0b6e6f925a3] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.8 K 2024-11-28T01:43:31,753 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0179181e703949f1b2d3060d1b8d996a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732758208432 2024-11-28T01:43:31,753 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a7d66a0be3954215874e4e0af9fa7a1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732758208519 2024-11-28T01:43:31,754 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting dff19389d1184a49a666b0b6e6f925a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732758210705 2024-11-28T01:43:31,766 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#603 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:31,766 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/6248f0ad04634ca783430253b7e309de is 50, key is test_row_0/C:col10/1732758210705/Put/seqid=0 2024-11-28T01:43:31,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742543_1719 (size=13153) 2024-11-28T01:43:31,775 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/6248f0ad04634ca783430253b7e309de as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6248f0ad04634ca783430253b7e309de 2024-11-28T01:43:31,779 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into 6248f0ad04634ca783430253b7e309de(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:31,779 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:31,779 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758211704; duration=0sec 2024-11-28T01:43:31,779 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:31,779 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:31,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-28T01:43:31,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:31,844 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:43:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:31,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b46ab49755ce41b4a2a20b1f547fb318_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758210839/Put/seqid=0 2024-11-28T01:43:31,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742544_1720 (size=12454) 2024-11-28T01:43:31,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:31,907 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b46ab49755ce41b4a2a20b1f547fb318_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b46ab49755ce41b4a2a20b1f547fb318_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:31,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e19be05694b540f6b979c5064a32fb81, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e19be05694b540f6b979c5064a32fb81 is 175, key is test_row_0/A:col10/1732758210839/Put/seqid=0 2024-11-28T01:43:31,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742545_1721 (size=31255) 2024-11-28T01:43:31,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e19be05694b540f6b979c5064a32fb81 2024-11-28T01:43:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/111377eefbf943b79e2a89b882b94779 is 50, key is test_row_0/B:col10/1732758210839/Put/seqid=0 2024-11-28T01:43:31,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742546_1722 (size=12301) 2024-11-28T01:43:31,949 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/111377eefbf943b79e2a89b882b94779 2024-11-28T01:43:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3ca84aa39b6045a49beb6fc89f9cac68 is 50, key is test_row_0/C:col10/1732758210839/Put/seqid=0 2024-11-28T01:43:31,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:32,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742547_1723 (size=12301) 2024-11-28T01:43:32,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758272033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758272035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758272036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758272038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T01:43:32,134 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#601 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:32,135 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/83ec2432321048f384c3821bf8a0cdab is 175, key is test_row_0/A:col10/1732758210705/Put/seqid=0 2024-11-28T01:43:32,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758272143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742548_1724 (size=32107) 2024-11-28T01:43:32,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758272152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758272152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758272152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,159 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/83ec2432321048f384c3821bf8a0cdab as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/83ec2432321048f384c3821bf8a0cdab 2024-11-28T01:43:32,168 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into 83ec2432321048f384c3821bf8a0cdab(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:32,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:32,168 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758211696; duration=0sec 2024-11-28T01:43:32,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:32,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758272353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758272363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758272363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758272363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3ca84aa39b6045a49beb6fc89f9cac68 2024-11-28T01:43:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/e19be05694b540f6b979c5064a32fb81 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e19be05694b540f6b979c5064a32fb81 2024-11-28T01:43:32,441 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e19be05694b540f6b979c5064a32fb81, entries=150, sequenceid=378, filesize=30.5 K 2024-11-28T01:43:32,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/111377eefbf943b79e2a89b882b94779 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/111377eefbf943b79e2a89b882b94779 2024-11-28T01:43:32,447 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/111377eefbf943b79e2a89b882b94779, entries=150, sequenceid=378, filesize=12.0 K 2024-11-28T01:43:32,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3ca84aa39b6045a49beb6fc89f9cac68 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3ca84aa39b6045a49beb6fc89f9cac68 2024-11-28T01:43:32,451 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3ca84aa39b6045a49beb6fc89f9cac68, entries=150, sequenceid=378, filesize=12.0 K 2024-11-28T01:43:32,452 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b33d90fd8529f8494aea715fa8bbbcde in 608ms, sequenceid=378, compaction requested=false 2024-11-28T01:43:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-28T01:43:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-28T01:43:32,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-28T01:43:32,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5280 sec 2024-11-28T01:43:32,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.5330 sec 2024-11-28T01:43:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:32,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:43:32,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:32,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:32,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:32,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:32,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:32,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:32,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112820032cfd4bb5468b97ccea8289e2d839_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758212035/Put/seqid=0 2024-11-28T01:43:32,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742549_1725 (size=12454) 2024-11-28T01:43:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758272685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:32,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758272689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112820032cfd4bb5468b97ccea8289e2d839_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112820032cfd4bb5468b97ccea8289e2d839_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:32,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/4e39a59dfeba4c0faafda4de6d54f5a2, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758272691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/4e39a59dfeba4c0faafda4de6d54f5a2 is 175, key is test_row_0/A:col10/1732758212035/Put/seqid=0 2024-11-28T01:43:32,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758272693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742550_1726 (size=31255) 2024-11-28T01:43:32,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758272803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758272803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758272803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:32,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758272804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:32,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758273012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758273012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758273012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758273012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T01:43:33,073 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:33,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-28T01:43:33,076 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T01:43:33,077 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:33,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:33,104 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/4e39a59dfeba4c0faafda4de6d54f5a2 2024-11-28T01:43:33,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/a19ff0d763ff4865ad154720e6a1c404 is 50, key is test_row_0/B:col10/1732758212035/Put/seqid=0 2024-11-28T01:43:33,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742551_1727 (size=12301) 2024-11-28T01:43:33,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/a19ff0d763ff4865ad154720e6a1c404 2024-11-28T01:43:33,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/9519f1573075481194710fb478115b1f is 50, key is test_row_0/C:col10/1732758212035/Put/seqid=0 2024-11-28T01:43:33,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742552_1728 (size=12301) 2024-11-28T01:43:33,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/9519f1573075481194710fb478115b1f 2024-11-28T01:43:33,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/4e39a59dfeba4c0faafda4de6d54f5a2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/4e39a59dfeba4c0faafda4de6d54f5a2 2024-11-28T01:43:33,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/4e39a59dfeba4c0faafda4de6d54f5a2, entries=150, sequenceid=399, filesize=30.5 K 2024-11-28T01:43:33,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/a19ff0d763ff4865ad154720e6a1c404 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a19ff0d763ff4865ad154720e6a1c404 2024-11-28T01:43:33,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a19ff0d763ff4865ad154720e6a1c404, entries=150, sequenceid=399, filesize=12.0 K 2024-11-28T01:43:33,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/9519f1573075481194710fb478115b1f as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/9519f1573075481194710fb478115b1f 2024-11-28T01:43:33,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/9519f1573075481194710fb478115b1f, entries=150, sequenceid=399, filesize=12.0 K 2024-11-28T01:43:33,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for b33d90fd8529f8494aea715fa8bbbcde in 504ms, sequenceid=399, compaction requested=true 2024-11-28T01:43:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:33,168 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T01:43:33,168 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:33,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:33,169 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:33,169 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:33,169 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/B is initiating minor compaction (all files) 2024-11-28T01:43:33,169 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/A is initiating minor compaction (all files) 2024-11-28T01:43:33,169 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/B in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:33,170 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/A in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:33,170 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aba4107402b5453aaf9b7ae8e5c9c221, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/111377eefbf943b79e2a89b882b94779, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a19ff0d763ff4865ad154720e6a1c404] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.9 K 2024-11-28T01:43:33,170 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/83ec2432321048f384c3821bf8a0cdab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e19be05694b540f6b979c5064a32fb81, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/4e39a59dfeba4c0faafda4de6d54f5a2] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=92.4 K 2024-11-28T01:43:33,170 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:33,170 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. files: [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/83ec2432321048f384c3821bf8a0cdab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e19be05694b540f6b979c5064a32fb81, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/4e39a59dfeba4c0faafda4de6d54f5a2] 2024-11-28T01:43:33,170 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting aba4107402b5453aaf9b7ae8e5c9c221, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732758210705 2024-11-28T01:43:33,171 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ec2432321048f384c3821bf8a0cdab, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732758210705 2024-11-28T01:43:33,171 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting 111377eefbf943b79e2a89b882b94779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732758210838 2024-11-28T01:43:33,171 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e19be05694b540f6b979c5064a32fb81, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732758210838 2024-11-28T01:43:33,171 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] compactions.Compactor(225): Compacting a19ff0d763ff4865ad154720e6a1c404, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732758212032 2024-11-28T01:43:33,171 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e39a59dfeba4c0faafda4de6d54f5a2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732758212032 2024-11-28T01:43:33,179 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:33,180 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#B#compaction#610 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:33,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411281e615fa256054dd2a346303d56c6c73d_b33d90fd8529f8494aea715fa8bbbcde store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:33,181 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/81aba075f0d9487ab7397e5f4ee6e93e is 50, key is test_row_0/B:col10/1732758212035/Put/seqid=0 2024-11-28T01:43:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T01:43:33,185 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411281e615fa256054dd2a346303d56c6c73d_b33d90fd8529f8494aea715fa8bbbcde, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:33,185 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281e615fa256054dd2a346303d56c6c73d_b33d90fd8529f8494aea715fa8bbbcde because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:33,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742553_1729 (size=13255) 2024-11-28T01:43:33,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742554_1730 (size=4469) 2024-11-28T01:43:33,204 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#A#compaction#611 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:33,205 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/a83e9ca69e6243a995aca60dcc9f4de5 is 175, key is test_row_0/A:col10/1732758212035/Put/seqid=0 2024-11-28T01:43:33,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742555_1731 (size=32209) 2024-11-28T01:43:33,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-28T01:43:33,232 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/a83e9ca69e6243a995aca60dcc9f4de5 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/a83e9ca69e6243a995aca60dcc9f4de5 2024-11-28T01:43:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:33,233 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:43:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:33,237 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/A of b33d90fd8529f8494aea715fa8bbbcde into a83e9ca69e6243a995aca60dcc9f4de5(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:33,237 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:33,237 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/A, priority=13, startTime=1732758213168; duration=0sec 2024-11-28T01:43:33,237 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T01:43:33,237 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:33,237 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T01:43:33,239 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T01:43:33,239 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1541): b33d90fd8529f8494aea715fa8bbbcde/C is initiating minor compaction (all files) 2024-11-28T01:43:33,239 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b33d90fd8529f8494aea715fa8bbbcde/C in TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:33,239 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6248f0ad04634ca783430253b7e309de, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3ca84aa39b6045a49beb6fc89f9cac68, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/9519f1573075481194710fb478115b1f] into tmpdir=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp, totalSize=36.9 K 2024-11-28T01:43:33,240 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6248f0ad04634ca783430253b7e309de, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732758210705 2024-11-28T01:43:33,241 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ca84aa39b6045a49beb6fc89f9cac68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732758210838 2024-11-28T01:43:33,242 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9519f1573075481194710fb478115b1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732758212032 2024-11-28T01:43:33,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128063a4def0609444ba209c6d1009fa240_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758212690/Put/seqid=0 2024-11-28T01:43:33,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742556_1732 (size=12454) 2024-11-28T01:43:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:33,266 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b33d90fd8529f8494aea715fa8bbbcde#C#compaction#613 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T01:43:33,266 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/f11ca3a9de48475ba9faded04855d8bb is 50, key is test_row_0/C:col10/1732758212035/Put/seqid=0 2024-11-28T01:43:33,269 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128063a4def0609444ba209c6d1009fa240_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128063a4def0609444ba209c6d1009fa240_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:33,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/28fe89bfb798445eb03a4c59930fc466, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:33,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/28fe89bfb798445eb03a4c59930fc466 is 175, key is test_row_0/A:col10/1732758212690/Put/seqid=0 2024-11-28T01:43:33,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742558_1734 (size=31255) 2024-11-28T01:43:33,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742557_1733 (size=13255) 2024-11-28T01:43:33,304 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/f11ca3a9de48475ba9faded04855d8bb as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f11ca3a9de48475ba9faded04855d8bb 2024-11-28T01:43:33,308 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/C of b33d90fd8529f8494aea715fa8bbbcde into f11ca3a9de48475ba9faded04855d8bb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:33,308 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:33,308 INFO [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/C, priority=13, startTime=1732758213168; duration=0sec 2024-11-28T01:43:33,308 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:33,308 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:33,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758273353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758273355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758273356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758273357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T01:43:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758273463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758273472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758273473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758273473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,598 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/81aba075f0d9487ab7397e5f4ee6e93e as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/81aba075f0d9487ab7397e5f4ee6e93e 2024-11-28T01:43:33,603 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b33d90fd8529f8494aea715fa8bbbcde/B of b33d90fd8529f8494aea715fa8bbbcde into 81aba075f0d9487ab7397e5f4ee6e93e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T01:43:33,603 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:33,603 INFO [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., storeName=b33d90fd8529f8494aea715fa8bbbcde/B, priority=13, startTime=1732758213168; duration=0sec 2024-11-28T01:43:33,603 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:33,603 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:33,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758273673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758273682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758273682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758273682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,691 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/28fe89bfb798445eb03a4c59930fc466 2024-11-28T01:43:33,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/1910fed969e44062bef666b4849cec86 is 50, key is test_row_0/B:col10/1732758212690/Put/seqid=0 2024-11-28T01:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T01:43:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742559_1735 (size=12301) 2024-11-28T01:43:33,705 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/1910fed969e44062bef666b4849cec86 2024-11-28T01:43:33,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3f6ff9385d7f41d29819ebe564fb4839 is 50, key is test_row_0/C:col10/1732758212690/Put/seqid=0 2024-11-28T01:43:33,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742560_1736 (size=12301) 2024-11-28T01:43:33,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758273983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758273993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758273993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:33,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758273993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,126 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3f6ff9385d7f41d29819ebe564fb4839 2024-11-28T01:43:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/28fe89bfb798445eb03a4c59930fc466 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/28fe89bfb798445eb03a4c59930fc466 2024-11-28T01:43:34,137 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/28fe89bfb798445eb03a4c59930fc466, entries=150, sequenceid=412, filesize=30.5 K 2024-11-28T01:43:34,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/1910fed969e44062bef666b4849cec86 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1910fed969e44062bef666b4849cec86 2024-11-28T01:43:34,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1910fed969e44062bef666b4849cec86, entries=150, sequenceid=412, filesize=12.0 K 2024-11-28T01:43:34,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3f6ff9385d7f41d29819ebe564fb4839 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3f6ff9385d7f41d29819ebe564fb4839 2024-11-28T01:43:34,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3f6ff9385d7f41d29819ebe564fb4839, entries=150, sequenceid=412, filesize=12.0 K 2024-11-28T01:43:34,150 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for b33d90fd8529f8494aea715fa8bbbcde in 917ms, sequenceid=412, compaction requested=false 2024-11-28T01:43:34,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:34,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:34,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-28T01:43:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-28T01:43:34,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-28T01:43:34,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0740 sec 2024-11-28T01:43:34,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.0790 sec 2024-11-28T01:43:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-28T01:43:34,212 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:34,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T01:43:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-28T01:43:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T01:43:34,214 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T01:43:34,224 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T01:43:34,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T01:43:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T01:43:34,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-28T01:43:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:34,377 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T01:43:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:34,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d735049ee6684bc8879b67064dfb1371_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758213350/Put/seqid=0 2024-11-28T01:43:34,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742561_1737 (size=12454) 2024-11-28T01:43:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:34,419 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d735049ee6684bc8879b67064dfb1371_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d735049ee6684bc8879b67064dfb1371_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:34,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8ccc6907e3a64bad97d8cd709bd9d0b2, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:34,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8ccc6907e3a64bad97d8cd709bd9d0b2 is 175, key is test_row_0/A:col10/1732758213350/Put/seqid=0 2024-11-28T01:43:34,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742562_1738 (size=31255) 2024-11-28T01:43:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. as already flushing 2024-11-28T01:43:34,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34508 deadline: 1732758274512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758274514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758274515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758274517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T01:43:34,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758274623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758274623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758274632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758274833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758274833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T01:43:34,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758274842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:34,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8ccc6907e3a64bad97d8cd709bd9d0b2 2024-11-28T01:43:34,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/4065bed072514b55aedacb079efc12c9 is 50, key is test_row_0/B:col10/1732758213350/Put/seqid=0 2024-11-28T01:43:34,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742563_1739 (size=12301) 2024-11-28T01:43:34,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/4065bed072514b55aedacb079efc12c9 2024-11-28T01:43:34,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/359a845588954a6caf5eb8b22a89f73b is 50, key is test_row_0/C:col10/1732758213350/Put/seqid=0 2024-11-28T01:43:34,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742564_1740 (size=12301) 2024-11-28T01:43:35,083 INFO [Thread-2725 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2725. 2024-11-28T01:43:35,083 DEBUG [Thread-2725 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,083 DEBUG [Thread-2725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,083 DEBUG [Thread-2725 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,083 INFO [Thread-2721 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2721. 2024-11-28T01:43:35,084 DEBUG [Thread-2721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,084 DEBUG [Thread-2721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,084 DEBUG [Thread-2721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,085 INFO [Thread-2723 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2723. 2024-11-28T01:43:35,085 DEBUG [Thread-2723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,085 DEBUG [Thread-2723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,085 DEBUG [Thread-2723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,087 INFO [Thread-2717 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2717. 2024-11-28T01:43:35,087 DEBUG [Thread-2717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,087 DEBUG [Thread-2717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,087 DEBUG [Thread-2717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,088 INFO [Thread-2719 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2719. 2024-11-28T01:43:35,088 DEBUG [Thread-2719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,088 DEBUG [Thread-2719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,089 DEBUG [Thread-2719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:35,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34502 deadline: 1732758275142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34470 deadline: 1732758275142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:35,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:35,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:35,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:35,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:35,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:35,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:35,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:43:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34486 deadline: 1732758275153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:35,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 , the old value is region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T01:43:35,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., hostname=9df3f25f5867,42371,1732758050017, seqNum=5 because the exception is null or not the one we care about 2024-11-28T01:43:35,342 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/359a845588954a6caf5eb8b22a89f73b 2024-11-28T01:43:35,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/8ccc6907e3a64bad97d8cd709bd9d0b2 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8ccc6907e3a64bad97d8cd709bd9d0b2 2024-11-28T01:43:35,349 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8ccc6907e3a64bad97d8cd709bd9d0b2, entries=150, sequenceid=435, filesize=30.5 K 2024-11-28T01:43:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/4065bed072514b55aedacb079efc12c9 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/4065bed072514b55aedacb079efc12c9 2024-11-28T01:43:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T01:43:35,353 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/4065bed072514b55aedacb079efc12c9, entries=150, sequenceid=435, filesize=12.0 K 2024-11-28T01:43:35,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/359a845588954a6caf5eb8b22a89f73b as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/359a845588954a6caf5eb8b22a89f73b 2024-11-28T01:43:35,357 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/359a845588954a6caf5eb8b22a89f73b, entries=150, sequenceid=435, filesize=12.0 K 2024-11-28T01:43:35,357 INFO [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b33d90fd8529f8494aea715fa8bbbcde in 980ms, sequenceid=435, compaction requested=true 2024-11-28T01:43:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-28T01:43:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-28T01:43:35,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-28T01:43:35,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1350 sec 2024-11-28T01:43:35,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.1490 sec 2024-11-28T01:43:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(8855): Flush requested on b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:35,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T01:43:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:35,534 INFO [Thread-2706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2706. 2024-11-28T01:43:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:35,534 DEBUG [Thread-2706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:35,534 DEBUG [Thread-2706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:35,534 DEBUG [Thread-2706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ebfb036ce77547f3aad33bfb966d92c0_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758214515/Put/seqid=0 2024-11-28T01:43:35,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742565_1741 (size=12454) 2024-11-28T01:43:35,654 INFO [Thread-2714 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2714. 2024-11-28T01:43:35,654 DEBUG [Thread-2714 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,654 DEBUG [Thread-2714 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,654 DEBUG [Thread-2714 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,654 INFO [Thread-2710 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2710. 2024-11-28T01:43:35,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,654 DEBUG [Thread-2710 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,654 DEBUG [Thread-2710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,654 INFO [Thread-2712 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2712. 2024-11-28T01:43:35,655 DEBUG [Thread-2712 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,655 DEBUG [Thread-2710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,655 DEBUG [Thread-2712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,655 DEBUG [Thread-2712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,664 INFO [Thread-2708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2708. 2024-11-28T01:43:35,664 DEBUG [Thread-2708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T01:43:35,664 DEBUG [Thread-2708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,665 DEBUG [Thread-2708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:35,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:35,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:35,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ebfb036ce77547f3aad33bfb966d92c0_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ebfb036ce77547f3aad33bfb966d92c0_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:35,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2f9c8fda2df74db2a519e535e65a016c, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:35,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2f9c8fda2df74db2a519e535e65a016c is 175, key is test_row_0/A:col10/1732758214515/Put/seqid=0 2024-11-28T01:43:35,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742566_1742 (size=31255) 2024-11-28T01:43:36,351 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2f9c8fda2df74db2a519e535e65a016c 2024-11-28T01:43:36,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f3bd33bcf86244b7b2b9e71d92e481ce is 50, key is test_row_0/B:col10/1732758214515/Put/seqid=0 2024-11-28T01:43:36,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742567_1743 (size=12301) 2024-11-28T01:43:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T01:43:36,362 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3032 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2908 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2892 2024-11-28T01:43:36,362 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3029 2024-11-28T01:43:36,363 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2893 2024-11-28T01:43:36,363 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T01:43:36,363 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:43:36,363 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:43:36,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:36,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:36,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:36,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T01:43:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T01:43:36,366 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758216366"}]},"ts":"1732758216366"} 2024-11-28T01:43:36,367 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T01:43:36,368 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T01:43:36,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T01:43:36,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, UNASSIGN}] 2024-11-28T01:43:36,370 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, UNASSIGN 2024-11-28T01:43:36,370 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=CLOSING, regionLocation=9df3f25f5867,42371,1732758050017 2024-11-28T01:43:36,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, UNASSIGN because future has completed 2024-11-28T01:43:36,372 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T01:43:36,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017}] 2024-11-28T01:43:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T01:43:36,524 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(122): Close b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:36,524 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T01:43:36,524 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1722): Closing b33d90fd8529f8494aea715fa8bbbcde, disabling compactions & flushes 2024-11-28T01:43:36,524 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T01:43:36,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f3bd33bcf86244b7b2b9e71d92e481ce 2024-11-28T01:43:36,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3bb01bc539dc4c8d97ed5a74cf0bc990 is 50, key is test_row_0/C:col10/1732758214515/Put/seqid=0 2024-11-28T01:43:36,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742568_1744 (size=12301) 2024-11-28T01:43:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T01:43:37,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3bb01bc539dc4c8d97ed5a74cf0bc990 2024-11-28T01:43:37,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/2f9c8fda2df74db2a519e535e65a016c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2f9c8fda2df74db2a519e535e65a016c 2024-11-28T01:43:37,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2f9c8fda2df74db2a519e535e65a016c, entries=150, sequenceid=448, filesize=30.5 K 2024-11-28T01:43:37,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/f3bd33bcf86244b7b2b9e71d92e481ce as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3bd33bcf86244b7b2b9e71d92e481ce 2024-11-28T01:43:37,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3bd33bcf86244b7b2b9e71d92e481ce, entries=150, sequenceid=448, filesize=12.0 K 2024-11-28T01:43:37,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/3bb01bc539dc4c8d97ed5a74cf0bc990 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3bb01bc539dc4c8d97ed5a74cf0bc990 2024-11-28T01:43:37,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3bb01bc539dc4c8d97ed5a74cf0bc990, entries=150, sequenceid=448, filesize=12.0 K 2024-11-28T01:43:37,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for b33d90fd8529f8494aea715fa8bbbcde in 1646ms, sequenceid=448, compaction requested=true 2024-11-28T01:43:37,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b33d90fd8529f8494aea715fa8bbbcde: 2024-11-28T01:43:37,181 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:37,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. because compaction request was cancelled 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. after waiting 0 ms 2024-11-28T01:43:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:37,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:A 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:37,181 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. because compaction request was cancelled 2024-11-28T01:43:37,181 DEBUG [RS:0;9df3f25f5867:42371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:B 2024-11-28T01:43:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b33d90fd8529f8494aea715fa8bbbcde:C, priority=-2147483648, current under compaction store size is 2 2024-11-28T01:43:37,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. because compaction request was cancelled 2024-11-28T01:43:37,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T01:43:37,181 DEBUG [RS:0;9df3f25f5867:42371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b33d90fd8529f8494aea715fa8bbbcde:C 2024-11-28T01:43:37,181 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(2902): Flushing b33d90fd8529f8494aea715fa8bbbcde 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=A 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=B 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b33d90fd8529f8494aea715fa8bbbcde, store=C 2024-11-28T01:43:37,181 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T01:43:37,185 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281efd87f2acd5469da1de38a641442d74_b33d90fd8529f8494aea715fa8bbbcde is 50, key is test_row_0/A:col10/1732758215653/Put/seqid=0 2024-11-28T01:43:37,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742569_1745 (size=12454) 2024-11-28T01:43:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T01:43:37,588 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:43:37,591 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281efd87f2acd5469da1de38a641442d74_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281efd87f2acd5469da1de38a641442d74_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:37,592 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/d303bcd6a4dd4a26af17b5a189654b0c, store: [table=TestAcidGuarantees family=A region=b33d90fd8529f8494aea715fa8bbbcde] 2024-11-28T01:43:37,593 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/d303bcd6a4dd4a26af17b5a189654b0c is 175, key is test_row_0/A:col10/1732758215653/Put/seqid=0 2024-11-28T01:43:37,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742570_1746 (size=31255) 2024-11-28T01:43:37,996 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/d303bcd6a4dd4a26af17b5a189654b0c 2024-11-28T01:43:38,006 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/dfe0819f534e4f90a4b0ef5c581cc0ef is 50, key is test_row_0/B:col10/1732758215653/Put/seqid=0 2024-11-28T01:43:38,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742571_1747 (size=12301) 2024-11-28T01:43:38,410 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/dfe0819f534e4f90a4b0ef5c581cc0ef 2024-11-28T01:43:38,415 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/67fc7fcbf8e346499a4f041dc67148e8 is 50, key is test_row_0/C:col10/1732758215653/Put/seqid=0 2024-11-28T01:43:38,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742572_1748 (size=12301) 2024-11-28T01:43:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T01:43:38,818 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/67fc7fcbf8e346499a4f041dc67148e8 2024-11-28T01:43:38,822 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/A/d303bcd6a4dd4a26af17b5a189654b0c as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/d303bcd6a4dd4a26af17b5a189654b0c 2024-11-28T01:43:38,824 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/d303bcd6a4dd4a26af17b5a189654b0c, entries=150, sequenceid=455, filesize=30.5 K 2024-11-28T01:43:38,825 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/B/dfe0819f534e4f90a4b0ef5c581cc0ef as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/dfe0819f534e4f90a4b0ef5c581cc0ef 2024-11-28T01:43:38,827 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/dfe0819f534e4f90a4b0ef5c581cc0ef, entries=150, sequenceid=455, filesize=12.0 K 2024-11-28T01:43:38,828 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/.tmp/C/67fc7fcbf8e346499a4f041dc67148e8 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/67fc7fcbf8e346499a4f041dc67148e8 2024-11-28T01:43:38,830 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/67fc7fcbf8e346499a4f041dc67148e8, entries=150, sequenceid=455, filesize=12.0 K 2024-11-28T01:43:38,831 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b33d90fd8529f8494aea715fa8bbbcde in 1650ms, sequenceid=455, compaction requested=true 2024-11-28T01:43:38,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/24f1cce9f033475194bbffe6b95a9337, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/18cf0dd0c21d4005aef1a20bc48b5dd0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/fb26fc3375c14939a05a9cf6d61127e3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f860bfc2b5564a15b25b873b800b3239, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8538732fd0a345f291588d5556e787e2, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/40f13e2b8ebf4d7398886b9cf5f56bc0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e95e4baeb0534911873d4086c3823b0f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2b9050c43df048898caeb6d146185335, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/71e782846d41487cad7832bf04bcf78d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2ef6a7d1204f461487de65dbdc10c041, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/ea2b24bda0e847d884e3d69b6ae822db, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8dcab3658ca94e57ba09a645e29de09d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/de95cd320ba745c7a5c1fcf16b04e0a5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/3c7b5955c8804d069250c1d76fbd5547, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/34721c5213d948d99eba95b15fa0fce3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/c75b0e396cca427995e5005052dab369, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f37841d1174a42e1aef4197112f1cd13, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e73a145c783341db97f183cbeb4b0e26, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/7d241cd3599047c3907ee7a7c26370a6, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/950d8af49df14e17b790a84f4157c125, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8b90e0140a98453d8ff1ef842203d0f9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/1134b139a07e4fec8fba14b6c94fcfae, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/048a0653a4614d05908b58027d8ade14, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/cf5c2a9fb59842daab2a5af8e1d1a966, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/5f6a45a197924ed7868b7e019ee84cd8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/83ec2432321048f384c3821bf8a0cdab, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e19be05694b540f6b979c5064a32fb81, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/4e39a59dfeba4c0faafda4de6d54f5a2] to archive 2024-11-28T01:43:38,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:43:38,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/24f1cce9f033475194bbffe6b95a9337 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/24f1cce9f033475194bbffe6b95a9337 2024-11-28T01:43:38,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/18cf0dd0c21d4005aef1a20bc48b5dd0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/18cf0dd0c21d4005aef1a20bc48b5dd0 2024-11-28T01:43:38,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/fb26fc3375c14939a05a9cf6d61127e3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/fb26fc3375c14939a05a9cf6d61127e3 2024-11-28T01:43:38,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f860bfc2b5564a15b25b873b800b3239 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f860bfc2b5564a15b25b873b800b3239 2024-11-28T01:43:38,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8538732fd0a345f291588d5556e787e2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8538732fd0a345f291588d5556e787e2 2024-11-28T01:43:38,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/40f13e2b8ebf4d7398886b9cf5f56bc0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/40f13e2b8ebf4d7398886b9cf5f56bc0 2024-11-28T01:43:38,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e95e4baeb0534911873d4086c3823b0f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e95e4baeb0534911873d4086c3823b0f 2024-11-28T01:43:38,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2b9050c43df048898caeb6d146185335 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2b9050c43df048898caeb6d146185335 2024-11-28T01:43:38,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/71e782846d41487cad7832bf04bcf78d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/71e782846d41487cad7832bf04bcf78d 2024-11-28T01:43:38,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2ef6a7d1204f461487de65dbdc10c041 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2ef6a7d1204f461487de65dbdc10c041 2024-11-28T01:43:38,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/ea2b24bda0e847d884e3d69b6ae822db to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/ea2b24bda0e847d884e3d69b6ae822db 2024-11-28T01:43:38,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8dcab3658ca94e57ba09a645e29de09d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8dcab3658ca94e57ba09a645e29de09d 2024-11-28T01:43:38,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/de95cd320ba745c7a5c1fcf16b04e0a5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/de95cd320ba745c7a5c1fcf16b04e0a5 2024-11-28T01:43:38,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/3c7b5955c8804d069250c1d76fbd5547 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/3c7b5955c8804d069250c1d76fbd5547 2024-11-28T01:43:38,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/34721c5213d948d99eba95b15fa0fce3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/34721c5213d948d99eba95b15fa0fce3 2024-11-28T01:43:38,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/c75b0e396cca427995e5005052dab369 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/c75b0e396cca427995e5005052dab369 2024-11-28T01:43:38,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f37841d1174a42e1aef4197112f1cd13 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/f37841d1174a42e1aef4197112f1cd13 2024-11-28T01:43:38,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e73a145c783341db97f183cbeb4b0e26 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e73a145c783341db97f183cbeb4b0e26 2024-11-28T01:43:38,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/7d241cd3599047c3907ee7a7c26370a6 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/7d241cd3599047c3907ee7a7c26370a6 2024-11-28T01:43:38,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/950d8af49df14e17b790a84f4157c125 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/950d8af49df14e17b790a84f4157c125 2024-11-28T01:43:38,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8b90e0140a98453d8ff1ef842203d0f9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8b90e0140a98453d8ff1ef842203d0f9 2024-11-28T01:43:38,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/1134b139a07e4fec8fba14b6c94fcfae to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/1134b139a07e4fec8fba14b6c94fcfae 2024-11-28T01:43:38,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/048a0653a4614d05908b58027d8ade14 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/048a0653a4614d05908b58027d8ade14 2024-11-28T01:43:38,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/cf5c2a9fb59842daab2a5af8e1d1a966 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/cf5c2a9fb59842daab2a5af8e1d1a966 2024-11-28T01:43:38,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/5f6a45a197924ed7868b7e019ee84cd8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/5f6a45a197924ed7868b7e019ee84cd8 2024-11-28T01:43:38,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/83ec2432321048f384c3821bf8a0cdab to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/83ec2432321048f384c3821bf8a0cdab 2024-11-28T01:43:38,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e19be05694b540f6b979c5064a32fb81 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/e19be05694b540f6b979c5064a32fb81 2024-11-28T01:43:38,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/4e39a59dfeba4c0faafda4de6d54f5a2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/4e39a59dfeba4c0faafda4de6d54f5a2 2024-11-28T01:43:38,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/2a8d65f6560745eb97fe98390840f719, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1d5ff9a2c4da425aa1e8dd1999c92dc1, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/3132e3e294c04572a2fb77dd762e0643, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a6e3f5e6c3954a4a931b50a8b180c267, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/04effb66d97847f982cc2afa3bea0eeb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f48e3fdcced840cebf74e99068342cdb, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/fc7d4a118e7b4260bc9292e79285f68f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/73b40078404c4b1baad108b983f74ef5, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/b7cc0856ddc44eb8beb5305070ccf507, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/cd490bbfc9634de0a42488318dd3c993, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3ea3c8c92cd42b4808ab0da1ce93103, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/97ce32a21d9647a6986e44b509b6c329, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/72ba9181d532498ab1873aa87035502d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/9cec2434c8d84c9fb6e6d75c5d248ebd, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/efac638cdabd4c66a3f142e8f5f3a532, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/0c7ae861bcf540e38b78dee69a6f7e88, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aeb55721c7de4b53ab1a87c5d59ec09b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/ba8409cb828b41a298ca0fbf842f4e35, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/51c800ba4ed94675ad8d3013227aaff0, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/18bb215416c44cd2a02b469a02ca6582, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/50fc4dea1a85434b90c00c30475412f8, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/78233b6878a1444e8b3fbddb8e154168, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/6e0785e8b7084339b8dc33474a47a258, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/64e77478affe4fa590889c26dde3e544, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aba4107402b5453aaf9b7ae8e5c9c221, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/8e3173980b814de0af21015a5b1d7b41, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/111377eefbf943b79e2a89b882b94779, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a19ff0d763ff4865ad154720e6a1c404] to archive 2024-11-28T01:43:38,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:43:38,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/2a8d65f6560745eb97fe98390840f719 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/2a8d65f6560745eb97fe98390840f719 2024-11-28T01:43:38,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1d5ff9a2c4da425aa1e8dd1999c92dc1 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1d5ff9a2c4da425aa1e8dd1999c92dc1 2024-11-28T01:43:38,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/3132e3e294c04572a2fb77dd762e0643 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/3132e3e294c04572a2fb77dd762e0643 2024-11-28T01:43:38,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a6e3f5e6c3954a4a931b50a8b180c267 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a6e3f5e6c3954a4a931b50a8b180c267 2024-11-28T01:43:38,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/04effb66d97847f982cc2afa3bea0eeb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/04effb66d97847f982cc2afa3bea0eeb 2024-11-28T01:43:38,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f48e3fdcced840cebf74e99068342cdb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f48e3fdcced840cebf74e99068342cdb 2024-11-28T01:43:38,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/fc7d4a118e7b4260bc9292e79285f68f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/fc7d4a118e7b4260bc9292e79285f68f 2024-11-28T01:43:38,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/73b40078404c4b1baad108b983f74ef5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/73b40078404c4b1baad108b983f74ef5 2024-11-28T01:43:38,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/b7cc0856ddc44eb8beb5305070ccf507 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/b7cc0856ddc44eb8beb5305070ccf507 2024-11-28T01:43:38,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/cd490bbfc9634de0a42488318dd3c993 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/cd490bbfc9634de0a42488318dd3c993 2024-11-28T01:43:38,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3ea3c8c92cd42b4808ab0da1ce93103 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3ea3c8c92cd42b4808ab0da1ce93103 2024-11-28T01:43:38,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/97ce32a21d9647a6986e44b509b6c329 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/97ce32a21d9647a6986e44b509b6c329 2024-11-28T01:43:38,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/72ba9181d532498ab1873aa87035502d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/72ba9181d532498ab1873aa87035502d 2024-11-28T01:43:38,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/9cec2434c8d84c9fb6e6d75c5d248ebd to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/9cec2434c8d84c9fb6e6d75c5d248ebd 2024-11-28T01:43:38,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/efac638cdabd4c66a3f142e8f5f3a532 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/efac638cdabd4c66a3f142e8f5f3a532 2024-11-28T01:43:38,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/0c7ae861bcf540e38b78dee69a6f7e88 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/0c7ae861bcf540e38b78dee69a6f7e88 2024-11-28T01:43:38,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aeb55721c7de4b53ab1a87c5d59ec09b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aeb55721c7de4b53ab1a87c5d59ec09b 2024-11-28T01:43:38,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/ba8409cb828b41a298ca0fbf842f4e35 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/ba8409cb828b41a298ca0fbf842f4e35 2024-11-28T01:43:38,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/51c800ba4ed94675ad8d3013227aaff0 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/51c800ba4ed94675ad8d3013227aaff0 2024-11-28T01:43:38,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/18bb215416c44cd2a02b469a02ca6582 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/18bb215416c44cd2a02b469a02ca6582 2024-11-28T01:43:38,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/50fc4dea1a85434b90c00c30475412f8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/50fc4dea1a85434b90c00c30475412f8 2024-11-28T01:43:38,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/78233b6878a1444e8b3fbddb8e154168 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/78233b6878a1444e8b3fbddb8e154168 2024-11-28T01:43:38,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/6e0785e8b7084339b8dc33474a47a258 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/6e0785e8b7084339b8dc33474a47a258 2024-11-28T01:43:38,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/64e77478affe4fa590889c26dde3e544 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/64e77478affe4fa590889c26dde3e544 2024-11-28T01:43:38,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aba4107402b5453aaf9b7ae8e5c9c221 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/aba4107402b5453aaf9b7ae8e5c9c221 2024-11-28T01:43:38,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/8e3173980b814de0af21015a5b1d7b41 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/8e3173980b814de0af21015a5b1d7b41 2024-11-28T01:43:38,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/111377eefbf943b79e2a89b882b94779 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/111377eefbf943b79e2a89b882b94779 2024-11-28T01:43:38,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a19ff0d763ff4865ad154720e6a1c404 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/a19ff0d763ff4865ad154720e6a1c404 2024-11-28T01:43:38,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4f2c4f788a614e27b2c44485d1009f3b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4e48adc9f1ca4d9381b6c4c635bbd40f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/77fbde1be275438d801dd9ffd6a07669, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3751d5ec080f468a99a4854ac4c71729, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d53a6b3a743f4cd4809f088259587259, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c5da3882788e42aca1bd90e48cc1968d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/66426595a1d24c6293253085b238aae3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/79543bc9f44347528d75d98ceff8dff9, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/148f178fb8c743079ae3b211433e479c, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d6937d280b2e48fb9aa2b2f9d30119b3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/1f96dc9998bf4e13b07095c4dd19db2f, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/39aebc621e6e4892b66e602b2032c468, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/eb6a124a65884f788f789baa51be0862, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6fd117ddc43d4fb3882295b955e0a793, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ce6127d115b04fe6b170656bf1953e17, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ece1f86cee844cf1b678dbf63f7d006d, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f6a5f442096146f8b2030b9064f9ab31, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/e720314297414d1f92cb976ede67d3f7, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/05ca0be05741443aba5d19f1da81075e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/31ee90b444cd4be38af8f4daf6f1a146, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ab2a714ffc304f19adce4e7f7f22849b, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/0179181e703949f1b2d3060d1b8d996a, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c98999c3e048487d9ebd9adec875ed51, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/a7d66a0be3954215874e4e0af9fa7a1e, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6248f0ad04634ca783430253b7e309de, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/dff19389d1184a49a666b0b6e6f925a3, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3ca84aa39b6045a49beb6fc89f9cac68, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/9519f1573075481194710fb478115b1f] to archive 2024-11-28T01:43:38,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T01:43:38,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4f2c4f788a614e27b2c44485d1009f3b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4f2c4f788a614e27b2c44485d1009f3b 2024-11-28T01:43:38,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4e48adc9f1ca4d9381b6c4c635bbd40f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/4e48adc9f1ca4d9381b6c4c635bbd40f 2024-11-28T01:43:38,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/77fbde1be275438d801dd9ffd6a07669 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/77fbde1be275438d801dd9ffd6a07669 2024-11-28T01:43:38,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3751d5ec080f468a99a4854ac4c71729 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3751d5ec080f468a99a4854ac4c71729 2024-11-28T01:43:38,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d53a6b3a743f4cd4809f088259587259 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d53a6b3a743f4cd4809f088259587259 2024-11-28T01:43:38,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c5da3882788e42aca1bd90e48cc1968d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c5da3882788e42aca1bd90e48cc1968d 2024-11-28T01:43:38,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/66426595a1d24c6293253085b238aae3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/66426595a1d24c6293253085b238aae3 2024-11-28T01:43:38,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/79543bc9f44347528d75d98ceff8dff9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/79543bc9f44347528d75d98ceff8dff9 2024-11-28T01:43:38,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/148f178fb8c743079ae3b211433e479c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/148f178fb8c743079ae3b211433e479c 2024-11-28T01:43:38,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d6937d280b2e48fb9aa2b2f9d30119b3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/d6937d280b2e48fb9aa2b2f9d30119b3 2024-11-28T01:43:38,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/1f96dc9998bf4e13b07095c4dd19db2f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/1f96dc9998bf4e13b07095c4dd19db2f 2024-11-28T01:43:38,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/39aebc621e6e4892b66e602b2032c468 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/39aebc621e6e4892b66e602b2032c468 2024-11-28T01:43:38,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/eb6a124a65884f788f789baa51be0862 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/eb6a124a65884f788f789baa51be0862 2024-11-28T01:43:38,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6fd117ddc43d4fb3882295b955e0a793 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6fd117ddc43d4fb3882295b955e0a793 2024-11-28T01:43:38,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ce6127d115b04fe6b170656bf1953e17 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ce6127d115b04fe6b170656bf1953e17 2024-11-28T01:43:38,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ece1f86cee844cf1b678dbf63f7d006d to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ece1f86cee844cf1b678dbf63f7d006d 2024-11-28T01:43:38,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f6a5f442096146f8b2030b9064f9ab31 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f6a5f442096146f8b2030b9064f9ab31 2024-11-28T01:43:38,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/e720314297414d1f92cb976ede67d3f7 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/e720314297414d1f92cb976ede67d3f7 2024-11-28T01:43:38,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/05ca0be05741443aba5d19f1da81075e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/05ca0be05741443aba5d19f1da81075e 2024-11-28T01:43:38,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/31ee90b444cd4be38af8f4daf6f1a146 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/31ee90b444cd4be38af8f4daf6f1a146 2024-11-28T01:43:38,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ab2a714ffc304f19adce4e7f7f22849b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/ab2a714ffc304f19adce4e7f7f22849b 2024-11-28T01:43:38,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/0179181e703949f1b2d3060d1b8d996a to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/0179181e703949f1b2d3060d1b8d996a 2024-11-28T01:43:38,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c98999c3e048487d9ebd9adec875ed51 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/c98999c3e048487d9ebd9adec875ed51 2024-11-28T01:43:38,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/a7d66a0be3954215874e4e0af9fa7a1e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/a7d66a0be3954215874e4e0af9fa7a1e 2024-11-28T01:43:38,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6248f0ad04634ca783430253b7e309de to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/6248f0ad04634ca783430253b7e309de 2024-11-28T01:43:38,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/dff19389d1184a49a666b0b6e6f925a3 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/dff19389d1184a49a666b0b6e6f925a3 2024-11-28T01:43:38,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3ca84aa39b6045a49beb6fc89f9cac68 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3ca84aa39b6045a49beb6fc89f9cac68 2024-11-28T01:43:38,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/9519f1573075481194710fb478115b1f to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/9519f1573075481194710fb478115b1f 2024-11-28T01:43:38,906 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/recovered.edits/458.seqid, newMaxSeqId=458, maxSeqId=4 2024-11-28T01:43:38,906 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. 2024-11-28T01:43:38,907 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1676): Region close journal for b33d90fd8529f8494aea715fa8bbbcde: Waiting for close lock at 1732758216524Running coprocessor pre-close hooks at 1732758216524Disabling compacts and flushes for region at 1732758216524Disabling writes for close at 1732758217181 (+657 ms)Obtaining lock to block concurrent updates at 1732758217181Preparing flush snapshotting stores in b33d90fd8529f8494aea715fa8bbbcde at 1732758217181Finished memstore snapshotting TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732758217181Flushing stores of TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde. at 1732758217181Flushing b33d90fd8529f8494aea715fa8bbbcde/A: creating writer at 1732758217182 (+1 ms)Flushing mob file b33d90fd8529f8494aea715fa8bbbcde/A: appending metadata at 1732758217185 (+3 ms)Flushing mob file b33d90fd8529f8494aea715fa8bbbcde/A: closing flushed file at 1732758217185Flushing b33d90fd8529f8494aea715fa8bbbcde/A: appending metadata at 1732758217592 (+407 ms)Flushing b33d90fd8529f8494aea715fa8bbbcde/A: closing flushed file at 1732758217592Flushing b33d90fd8529f8494aea715fa8bbbcde/B: creating writer at 1732758217999 (+407 ms)Flushing b33d90fd8529f8494aea715fa8bbbcde/B: appending metadata at 1732758218006 (+7 ms)Flushing b33d90fd8529f8494aea715fa8bbbcde/B: closing flushed file at 1732758218006Flushing b33d90fd8529f8494aea715fa8bbbcde/C: creating writer at 1732758218413 (+407 ms)Flushing b33d90fd8529f8494aea715fa8bbbcde/C: appending metadata at 1732758218415 (+2 ms)Flushing b33d90fd8529f8494aea715fa8bbbcde/C: closing flushed file at 1732758218415Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b5cb912: reopening flushed file at 1732758218821 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44c9891d: reopening flushed file at 1732758218824 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7385b93c: reopening flushed file at 1732758218827 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b33d90fd8529f8494aea715fa8bbbcde in 1650ms, sequenceid=455, compaction requested=true at 1732758218831 (+4 ms)Writing region close event to WAL at 1732758218904 (+73 ms)Running coprocessor post-close hooks at 1732758218906 (+2 ms)Closed at 1732758218906 2024-11-28T01:43:38,908 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(157): Closed b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:38,909 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=b33d90fd8529f8494aea715fa8bbbcde, regionState=CLOSED 2024-11-28T01:43:38,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 because future has completed 2024-11-28T01:43:38,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-28T01:43:38,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; CloseRegionProcedure b33d90fd8529f8494aea715fa8bbbcde, server=9df3f25f5867,42371,1732758050017 in 2.5380 sec 2024-11-28T01:43:38,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=180, resume processing ppid=179 2024-11-28T01:43:38,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, ppid=179, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b33d90fd8529f8494aea715fa8bbbcde, UNASSIGN in 2.5430 sec 2024-11-28T01:43:38,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-28T01:43:38,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5450 sec 2024-11-28T01:43:38,915 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758218915"}]},"ts":"1732758218915"} 2024-11-28T01:43:38,916 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T01:43:38,916 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T01:43:38,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5530 sec 2024-11-28T01:43:39,630 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T01:43:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-28T01:43:40,523 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:40,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T01:43:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:40,525 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-28T01:43:40,525 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=182, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:40,528 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,529 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C, FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/recovered.edits] 2024-11-28T01:43:40,531 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/28fe89bfb798445eb03a4c59930fc466 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/28fe89bfb798445eb03a4c59930fc466 2024-11-28T01:43:40,532 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2f9c8fda2df74db2a519e535e65a016c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/2f9c8fda2df74db2a519e535e65a016c 2024-11-28T01:43:40,533 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8ccc6907e3a64bad97d8cd709bd9d0b2 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/8ccc6907e3a64bad97d8cd709bd9d0b2 2024-11-28T01:43:40,534 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/a83e9ca69e6243a995aca60dcc9f4de5 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/a83e9ca69e6243a995aca60dcc9f4de5 2024-11-28T01:43:40,534 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/d303bcd6a4dd4a26af17b5a189654b0c to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/A/d303bcd6a4dd4a26af17b5a189654b0c 2024-11-28T01:43:40,536 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1910fed969e44062bef666b4849cec86 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/1910fed969e44062bef666b4849cec86 2024-11-28T01:43:40,537 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/4065bed072514b55aedacb079efc12c9 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/4065bed072514b55aedacb079efc12c9 2024-11-28T01:43:40,537 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/81aba075f0d9487ab7397e5f4ee6e93e to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/81aba075f0d9487ab7397e5f4ee6e93e 2024-11-28T01:43:40,538 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/dfe0819f534e4f90a4b0ef5c581cc0ef to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/dfe0819f534e4f90a4b0ef5c581cc0ef 2024-11-28T01:43:40,539 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3bd33bcf86244b7b2b9e71d92e481ce to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/B/f3bd33bcf86244b7b2b9e71d92e481ce 2024-11-28T01:43:40,540 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/359a845588954a6caf5eb8b22a89f73b to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/359a845588954a6caf5eb8b22a89f73b 2024-11-28T01:43:40,541 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3bb01bc539dc4c8d97ed5a74cf0bc990 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3bb01bc539dc4c8d97ed5a74cf0bc990 2024-11-28T01:43:40,542 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3f6ff9385d7f41d29819ebe564fb4839 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/3f6ff9385d7f41d29819ebe564fb4839 2024-11-28T01:43:40,542 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/67fc7fcbf8e346499a4f041dc67148e8 to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/67fc7fcbf8e346499a4f041dc67148e8 2024-11-28T01:43:40,543 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f11ca3a9de48475ba9faded04855d8bb to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/C/f11ca3a9de48475ba9faded04855d8bb 2024-11-28T01:43:40,545 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/recovered.edits/458.seqid to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde/recovered.edits/458.seqid 2024-11-28T01:43:40,545 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/default/TestAcidGuarantees/b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,545 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T01:43:40,546 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T01:43:40,546 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T01:43:40,548 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128063a4def0609444ba209c6d1009fa240_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128063a4def0609444ba209c6d1009fa240_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,549 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112817c56812971742f6a5a26821a663e597_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112817c56812971742f6a5a26821a663e597_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,550 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128199009a454ad4c20bf5c2295035c187a_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128199009a454ad4c20bf5c2295035c187a_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,550 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281e6e977fb9714ba7865d7d3fca0b39ba_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281e6e977fb9714ba7865d7d3fca0b39ba_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,551 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281efd87f2acd5469da1de38a641442d74_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281efd87f2acd5469da1de38a641442d74_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,552 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112820032cfd4bb5468b97ccea8289e2d839_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112820032cfd4bb5468b97ccea8289e2d839_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,553 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128272b741811d640b880b81b1daa118186_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128272b741811d640b880b81b1daa118186_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,553 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283ad78459d7114d5ca0c61717d06de614_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283ad78459d7114d5ca0c61717d06de614_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,554 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283fbd614b5ea24e579b8aceb92e000651_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283fbd614b5ea24e579b8aceb92e000651_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,555 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128424c649a375d485fb1a6575f67378a09_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128424c649a375d485fb1a6575f67378a09_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,555 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112865e16c7c05f34a9881f0866c1a17c71d_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112865e16c7c05f34a9881f0866c1a17c71d_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,556 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411286d68e41d93dd4d60b45f9ab54a2ec10d_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411286d68e41d93dd4d60b45f9ab54a2ec10d_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112871410947969e46a7aacf89cfa737d1fd_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112871410947969e46a7aacf89cfa737d1fd_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,558 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287f206feb5b284b70bca66a481d5806a8_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287f206feb5b284b70bca66a481d5806a8_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,558 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288a3d50be72804d729359e731192e55d4_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288a3d50be72804d729359e731192e55d4_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,559 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a1388ac4a95e47e5b24e1552da926882_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a1388ac4a95e47e5b24e1552da926882_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,560 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a3088ab701b44adcb60c49d821f8ce1b_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a3088ab701b44adcb60c49d821f8ce1b_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,560 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b05cc2dee1a74e00aa10b49b15298a31_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b05cc2dee1a74e00aa10b49b15298a31_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b1b58f6b68d64e3aad4ac6b6248f5fa9_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b1b58f6b68d64e3aad4ac6b6248f5fa9_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,562 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b46ab49755ce41b4a2a20b1f547fb318_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b46ab49755ce41b4a2a20b1f547fb318_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,563 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c09fe669a74f445fa2aa92060b937d2a_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c09fe669a74f445fa2aa92060b937d2a_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,563 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d735049ee6684bc8879b67064dfb1371_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d735049ee6684bc8879b67064dfb1371_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128da74f51d62384f1794c738091f2c5f36_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128da74f51d62384f1794c738091f2c5f36_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ebfb036ce77547f3aad33bfb966d92c0_b33d90fd8529f8494aea715fa8bbbcde to hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ebfb036ce77547f3aad33bfb966d92c0_b33d90fd8529f8494aea715fa8bbbcde 2024-11-28T01:43:40,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T01:43:40,567 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=182, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:40,569 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T01:43:40,571 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T01:43:40,572 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=182, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:40,572 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T01:43:40,572 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732758220572"}]},"ts":"9223372036854775807"} 2024-11-28T01:43:40,573 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T01:43:40,573 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b33d90fd8529f8494aea715fa8bbbcde, NAME => 'TestAcidGuarantees,,1732758191972.b33d90fd8529f8494aea715fa8bbbcde.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T01:43:40,574 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T01:43:40,574 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732758220574"}]},"ts":"9223372036854775807"} 2024-11-28T01:43:40,575 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T01:43:40,575 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=182, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T01:43:40,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-28T01:43:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45903 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-28T01:43:40,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T01:43:40,633 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T01:43:40,643 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 248) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=507 (was 566), ProcessCount=11 (was 11), AvailableMemoryMB=6465 (was 6582) 2024-11-28T01:43:40,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T01:43:40,643 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:43:40,643 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:43:40,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:40,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:40,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:43:40,644 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T01:43:40,644 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1166144013, stopped=false 2024-11-28T01:43:40,644 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9df3f25f5867,45903,1732758049085 2024-11-28T01:43:40,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:43:40,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:43:40,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:43:40,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:43:40,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T01:43:40,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:43:40,647 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:43:40,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:43:40,647 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:43:40,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:40,647 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9df3f25f5867,42371,1732758050017' ***** 2024-11-28T01:43:40,648 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T01:43:40,648 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T01:43:40,648 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T01:43:40,648 INFO [RS:0;9df3f25f5867:42371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T01:43:40,648 INFO [RS:0;9df3f25f5867:42371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T01:43:40,649 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(959): stopping server 9df3f25f5867,42371,1732758050017 2024-11-28T01:43:40,649 INFO [RS:0;9df3f25f5867:42371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:43:40,649 INFO [RS:0;9df3f25f5867:42371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9df3f25f5867:42371. 2024-11-28T01:43:40,649 DEBUG [RS:0;9df3f25f5867:42371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:43:40,649 DEBUG [RS:0;9df3f25f5867:42371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:40,649 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T01:43:40,649 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T01:43:40,649 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T01:43:40,649 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T01:43:40,650 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T01:43:40,650 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T01:43:40,650 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T01:43:40,650 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T01:43:40,650 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T01:43:40,650 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T01:43:40,650 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T01:43:40,650 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:40,650 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-28T01:43:40,657 INFO [regionserver/9df3f25f5867:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:43:40,675 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/info/502a7732830f438c9b8796594ab9d2c6 is 95, key is TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285./info:/1732758078324/DeleteFamily/seqid=0 2024-11-28T01:43:40,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742573_1749 (size=6514) 2024-11-28T01:43:40,851 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:41,051 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:41,078 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/info/502a7732830f438c9b8796594ab9d2c6 2024-11-28T01:43:41,096 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/ns/369a5468b6284de49dff59b8c96e7303 is 93, key is TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285./ns:/1732758078262/DeleteFamily/seqid=0 2024-11-28T01:43:41,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742574_1750 (size=5906) 2024-11-28T01:43:41,251 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:41,451 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:41,483 INFO [regionserver/9df3f25f5867:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T01:43:41,483 INFO [regionserver/9df3f25f5867:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T01:43:41,499 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/ns/369a5468b6284de49dff59b8c96e7303 2024-11-28T01:43:41,524 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/rep_barrier/768b5cb37d7a43a88e0c148778007727 is 102, key is TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285./rep_barrier:/1732758078262/DeleteFamily/seqid=0 2024-11-28T01:43:41,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742575_1751 (size=6025) 2024-11-28T01:43:41,651 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:41,852 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:41,927 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/rep_barrier/768b5cb37d7a43a88e0c148778007727 2024-11-28T01:43:41,947 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/table/37cb276cd80b474992776a6fcfaa3fda is 96, key is TestAcidGuarantees,,1732758052673.707ae842769f4709bfcd7372850df285./table:/1732758078262/DeleteFamily/seqid=0 2024-11-28T01:43:41,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742576_1752 (size=5936) 2024-11-28T01:43:42,052 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:42,252 DEBUG [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:43:42,350 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/table/37cb276cd80b474992776a6fcfaa3fda 2024-11-28T01:43:42,354 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/info/502a7732830f438c9b8796594ab9d2c6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/info/502a7732830f438c9b8796594ab9d2c6 2024-11-28T01:43:42,356 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/info/502a7732830f438c9b8796594ab9d2c6, entries=12, sequenceid=90, filesize=6.4 K 2024-11-28T01:43:42,357 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/ns/369a5468b6284de49dff59b8c96e7303 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/ns/369a5468b6284de49dff59b8c96e7303 2024-11-28T01:43:42,359 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/ns/369a5468b6284de49dff59b8c96e7303, entries=8, sequenceid=90, filesize=5.8 K 2024-11-28T01:43:42,360 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/rep_barrier/768b5cb37d7a43a88e0c148778007727 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/rep_barrier/768b5cb37d7a43a88e0c148778007727 2024-11-28T01:43:42,362 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/rep_barrier/768b5cb37d7a43a88e0c148778007727, entries=6, sequenceid=90, filesize=5.9 K 2024-11-28T01:43:42,363 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/.tmp/table/37cb276cd80b474992776a6fcfaa3fda as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/table/37cb276cd80b474992776a6fcfaa3fda 2024-11-28T01:43:42,365 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/table/37cb276cd80b474992776a6fcfaa3fda, entries=7, sequenceid=90, filesize=5.8 K 2024-11-28T01:43:42,366 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1716ms, sequenceid=90, compaction requested=false 2024-11-28T01:43:42,370 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-28T01:43:42,371 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T01:43:42,371 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T01:43:42,371 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732758220650Running coprocessor pre-close hooks at 1732758220650Disabling compacts and flushes for region at 1732758220650Disabling writes for close at 1732758220650Obtaining lock to block concurrent updates at 1732758220650Preparing flush snapshotting stores in 1588230740 at 1732758220650Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732758220651 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732758220651Flushing 1588230740/info: creating writer at 1732758220652 (+1 ms)Flushing 1588230740/info: appending metadata at 1732758220674 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732758220674Flushing 1588230740/ns: creating writer at 1732758221082 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732758221096 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732758221096Flushing 1588230740/rep_barrier: creating writer at 1732758221503 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732758221523 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732758221523Flushing 1588230740/table: creating writer at 1732758221930 (+407 ms)Flushing 1588230740/table: appending metadata at 1732758221946 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732758221946Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22b7fc90: reopening flushed file at 1732758222353 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11672bc2: reopening flushed file at 1732758222356 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ebaae52: reopening flushed file at 1732758222359 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@111d5412: reopening flushed file at 1732758222362 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1716ms, sequenceid=90, compaction requested=false at 1732758222366 (+4 ms)Writing region close event to WAL at 1732758222368 (+2 ms)Running coprocessor post-close hooks at 1732758222371 (+3 ms)Closed at 1732758222371 2024-11-28T01:43:42,371 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T01:43:42,453 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(976): stopping server 9df3f25f5867,42371,1732758050017; all regions closed. 2024-11-28T01:43:42,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741834_1010 (size=25344) 2024-11-28T01:43:42,459 DEBUG [RS:0;9df3f25f5867:42371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/oldWALs 2024-11-28T01:43:42,459 INFO [RS:0;9df3f25f5867:42371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C42371%2C1732758050017.meta:.meta(num 1732758052190) 2024-11-28T01:43:42,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741833_1009 (size=21516005) 2024-11-28T01:43:42,462 DEBUG [RS:0;9df3f25f5867:42371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/oldWALs 2024-11-28T01:43:42,462 INFO [RS:0;9df3f25f5867:42371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C42371%2C1732758050017:(num 1732758051559) 2024-11-28T01:43:42,462 DEBUG [RS:0;9df3f25f5867:42371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:43:42,462 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:43:42,463 INFO [RS:0;9df3f25f5867:42371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:43:42,463 INFO [RS:0;9df3f25f5867:42371 {}] hbase.ChoreService(370): Chore service for: regionserver/9df3f25f5867:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T01:43:42,463 INFO [RS:0;9df3f25f5867:42371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:43:42,463 INFO [regionserver/9df3f25f5867:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:43:42,464 INFO [RS:0;9df3f25f5867:42371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42371 2024-11-28T01:43:42,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9df3f25f5867,42371,1732758050017 2024-11-28T01:43:42,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:43:42,468 INFO [RS:0;9df3f25f5867:42371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:43:42,470 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9df3f25f5867,42371,1732758050017] 2024-11-28T01:43:42,472 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9df3f25f5867,42371,1732758050017 already deleted, retry=false 2024-11-28T01:43:42,472 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9df3f25f5867,42371,1732758050017 expired; onlineServers=0 2024-11-28T01:43:42,472 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9df3f25f5867,45903,1732758049085' ***** 2024-11-28T01:43:42,472 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T01:43:42,472 INFO [M:0;9df3f25f5867:45903 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:43:42,472 INFO [M:0;9df3f25f5867:45903 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:43:42,472 DEBUG [M:0;9df3f25f5867:45903 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T01:43:42,473 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T01:43:42,473 DEBUG [M:0;9df3f25f5867:45903 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T01:43:42,473 DEBUG [master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758051234 {}] cleaner.HFileCleaner(306): Exit Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758051234,5,FailOnTimeoutGroup] 2024-11-28T01:43:42,473 DEBUG [master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758051249 {}] cleaner.HFileCleaner(306): Exit Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758051249,5,FailOnTimeoutGroup] 2024-11-28T01:43:42,473 INFO [M:0;9df3f25f5867:45903 {}] hbase.ChoreService(370): Chore service for: master/9df3f25f5867:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T01:43:42,473 INFO [M:0;9df3f25f5867:45903 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:43:42,473 DEBUG [M:0;9df3f25f5867:45903 {}] master.HMaster(1795): Stopping service threads 2024-11-28T01:43:42,473 INFO [M:0;9df3f25f5867:45903 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T01:43:42,473 INFO [M:0;9df3f25f5867:45903 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T01:43:42,473 ERROR [M:0;9df3f25f5867:45903 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:35107 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35107,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-28T01:43:42,474 INFO [M:0;9df3f25f5867:45903 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T01:43:42,474 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T01:43:42,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T01:43:42,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:43:42,475 DEBUG [M:0;9df3f25f5867:45903 {}] zookeeper.ZKUtil(347): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T01:43:42,475 WARN [M:0;9df3f25f5867:45903 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T01:43:42,475 INFO [M:0;9df3f25f5867:45903 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/.lastflushedseqids 2024-11-28T01:43:42,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742577_1753 (size=72) 2024-11-28T01:43:42,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:43:42,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x1003bc946520001, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:43:42,571 INFO [RS:0;9df3f25f5867:42371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:43:42,572 INFO [RS:0;9df3f25f5867:42371 {}] regionserver.HRegionServer(1031): Exiting; stopping=9df3f25f5867,42371,1732758050017; zookeeper connection closed. 2024-11-28T01:43:42,572 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2beb323c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2beb323c 2024-11-28T01:43:42,572 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T01:43:42,885 INFO [M:0;9df3f25f5867:45903 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T01:43:42,885 INFO [M:0;9df3f25f5867:45903 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T01:43:42,885 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T01:43:42,885 INFO [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:43:42,885 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:43:42,885 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T01:43:42,885 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:43:42,885 INFO [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=836.03 KB heapSize=1.00 MB 2024-11-28T01:43:42,901 DEBUG [M:0;9df3f25f5867:45903 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e26520b046fc46379d8909b42997136a is 82, key is hbase:meta,,1/info:regioninfo/1732758052294/Put/seqid=0 2024-11-28T01:43:42,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742578_1754 (size=5672) 2024-11-28T01:43:43,305 INFO [M:0;9df3f25f5867:45903 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2389 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e26520b046fc46379d8909b42997136a 2024-11-28T01:43:43,325 DEBUG [M:0;9df3f25f5867:45903 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f86a7d28a355403c83eb21d0aa47dae6 is 2479, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1732758080665/Put/seqid=0 2024-11-28T01:43:43,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742579_1755 (size=47448) 2024-11-28T01:43:43,729 INFO [M:0;9df3f25f5867:45903 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=835.48 KB at sequenceid=2389 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f86a7d28a355403c83eb21d0aa47dae6 2024-11-28T01:43:43,732 INFO [M:0;9df3f25f5867:45903 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f86a7d28a355403c83eb21d0aa47dae6 2024-11-28T01:43:43,747 DEBUG [M:0;9df3f25f5867:45903 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bc1d59ca3cb841349538486304289d3d is 69, key is 9df3f25f5867,42371,1732758050017/rs:state/1732758051293/Put/seqid=0 2024-11-28T01:43:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073742580_1756 (size=5156) 2024-11-28T01:43:44,150 INFO [M:0;9df3f25f5867:45903 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2389 (bloomFilter=true), to=hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bc1d59ca3cb841349538486304289d3d 2024-11-28T01:43:44,154 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e26520b046fc46379d8909b42997136a as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e26520b046fc46379d8909b42997136a 2024-11-28T01:43:44,157 INFO [M:0;9df3f25f5867:45903 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e26520b046fc46379d8909b42997136a, entries=8, sequenceid=2389, filesize=5.5 K 2024-11-28T01:43:44,157 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f86a7d28a355403c83eb21d0aa47dae6 as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f86a7d28a355403c83eb21d0aa47dae6 2024-11-28T01:43:44,160 INFO [M:0;9df3f25f5867:45903 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f86a7d28a355403c83eb21d0aa47dae6 2024-11-28T01:43:44,160 INFO [M:0;9df3f25f5867:45903 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f86a7d28a355403c83eb21d0aa47dae6, entries=182, sequenceid=2389, filesize=46.3 K 2024-11-28T01:43:44,161 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bc1d59ca3cb841349538486304289d3d as hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bc1d59ca3cb841349538486304289d3d 2024-11-28T01:43:44,163 INFO [M:0;9df3f25f5867:45903 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35107/user/jenkins/test-data/85aeff63-e84c-23e4-4aca-ad5fbb96a6ec/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bc1d59ca3cb841349538486304289d3d, entries=1, sequenceid=2389, filesize=5.0 K 2024-11-28T01:43:44,164 INFO [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(3140): Finished flush of dataSize ~836.03 KB/856099, heapSize ~1.00 MB/1053096, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2389, compaction requested=false 2024-11-28T01:43:44,166 INFO [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:43:44,166 DEBUG [M:0;9df3f25f5867:45903 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732758222885Disabling compacts and flushes for region at 1732758222885Disabling writes for close at 1732758222885Obtaining lock to block concurrent updates at 1732758222885Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732758222885Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=856099, getHeapSize=1053336, getOffHeapSize=0, getCellsCount=2596 at 1732758222886 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732758222886Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732758222886Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732758222901 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732758222901Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732758223308 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732758223324 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732758223324Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732758223732 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732758223746 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732758223746Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5add3b1f: reopening flushed file at 1732758224153 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101d6c5e: reopening flushed file at 1732758224157 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7004bc62: reopening flushed file at 1732758224161 (+4 ms)Finished flush of dataSize ~836.03 KB/856099, heapSize ~1.00 MB/1053096, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2389, compaction requested=false at 1732758224164 (+3 ms)Writing region close event to WAL at 1732758224165 (+1 ms)Closed at 1732758224165 2024-11-28T01:43:44,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35065 is added to blk_1073741830_1006 (size=1012631) 2024-11-28T01:43:44,168 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:43:44,168 INFO [M:0;9df3f25f5867:45903 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T01:43:44,169 INFO [M:0;9df3f25f5867:45903 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45903 2024-11-28T01:43:44,169 INFO [M:0;9df3f25f5867:45903 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:43:44,271 INFO [M:0;9df3f25f5867:45903 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:43:44,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:43:44,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45903-0x1003bc946520000, quorum=127.0.0.1:63369, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:43:44,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T01:43:44,284 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T01:43:44,284 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T01:43:44,284 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T01:43:44,284 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/hadoop.log.dir/,STOPPED} 2024-11-28T01:43:44,287 WARN [BP-129411190-172.17.0.2-1732758045693 heartbeating to localhost/127.0.0.1:35107 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T01:43:44,287 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T01:43:44,287 WARN [BP-129411190-172.17.0.2-1732758045693 heartbeating to localhost/127.0.0.1:35107 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-129411190-172.17.0.2-1732758045693 (Datanode Uuid 5222bf73-d4c7-4850-a6fb-744c437557f8) service to localhost/127.0.0.1:35107 2024-11-28T01:43:44,287 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T01:43:44,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/data/data1/current/BP-129411190-172.17.0.2-1732758045693 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T01:43:44,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/cluster_25a1eec3-6245-d116-7977-d886a94f3a5d/data/data2/current/BP-129411190-172.17.0.2-1732758045693 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T01:43:44,291 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T01:43:44,299 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T01:43:44,300 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T01:43:44,300 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T01:43:44,301 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T01:43:44,301 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2cc79f7b-5681-9703-1916-439b577ef2e4/hadoop.log.dir/,STOPPED} 2024-11-28T01:43:44,319 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T01:43:44,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:43:44,428 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-28T01:43:44,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down